{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os.path as ops\n",
    "import numpy as np\n",
    "import torch\n",
    "import cv2\n",
    "import time\n",
    "from dataset.dataset_utils import TUSIMPLE, TUSIMPLE_AUG\n",
    "from Lanenet.model2 import Lanenet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build The datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_set length 6164\n",
      "valid_set length 362\n",
      "test_set length 726\n"
     ]
    }
   ],
   "source": [
    "# root = '/Users/smiffy/Documents/GitHub/TUSIMPLE/Data_Tusimple_PyTorch/training'\n",
    "root = 'TUSIMPLE/txt_for_local'\n",
    "train_set = TUSIMPLE_AUG(root=root, flag='train')\n",
    "valid_set = TUSIMPLE_AUG(root=root, flag='valid')\n",
    "test_set = TUSIMPLE_AUG(root=root, flag='test')\n",
    "\n",
    "print('train_set length {}'.format(len(train_set)))\n",
    "print('valid_set length {}'.format(len(valid_set)))\n",
    "print('test_set length {}'.format(len(test_set)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "image type <class 'torch.Tensor'>\n",
      "image size torch.Size([3, 256, 512]) \n",
      "\n",
      "gt binary image type <class 'torch.Tensor'>\n",
      "gt binary image size torch.Size([256, 512])\n",
      "items in gt binary image tensor([0, 1]) \n",
      "\n",
      "gt instance type <class 'torch.Tensor'>\n",
      "gt instance size torch.Size([256, 512])\n",
      "items in gt instance tensor([  0,  20,  70, 120, 170]) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "gt, bgt, igt = train_set[0]\n",
    "print('image type {}'.format(type(gt)))\n",
    "print('image size {} \\n'.format(gt.size()))\n",
    "\n",
    "print('gt binary image type {}'.format(type(bgt)))\n",
    "print('gt binary image size {}'.format(bgt.size()))\n",
    "print('items in gt binary image {} \\n'.format(torch.unique(bgt)))\n",
    "\n",
    "print('gt instance type {}'.format(type(igt)))\n",
    "print('gt instance size {}'.format(igt.size()))\n",
    "print('items in gt instance {} \\n'.format(torch.unique(igt)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 8\n",
    "\n",
    "data_loader_train = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "data_loader_valid = torch.utils.data.DataLoader(valid_set, batch_size=1, shuffle=True, num_workers=0)\n",
    "data_loader_test = torch.utils.data.DataLoader(test_set, batch_size=1, shuffle=False, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model and optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 5e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "\n",
    "LaneNet_model = Lanenet(2, 4)\n",
    "LaneNet_model.to(device)\n",
    "\n",
    "params = [p for p in LaneNet_model.parameters() if p.requires_grad]\n",
    "optimizer = torch.optim.Adam(params, lr=learning_rate, weight_decay=0.0002)\n",
    "\n",
    "lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Lanenet.cluster_loss3 import cluster_loss\n",
    "criterion = cluster_loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch[0] iter[0] loss: [0.8191455006599426, 9.361795425415039] \n",
      "epoch[0] iter[40] loss: [0.5152292847633362, 6.026969909667969] \n",
      "epoch[0] iter[80] loss: [0.39719387888908386, 3.175370931625366] \n",
      "epoch[0] iter[120] loss: [0.34216248989105225, 2.1009573936462402] \n",
      "epoch[0] iter[160] loss: [0.30980077385902405, 1.7131037712097168] \n",
      "epoch[0] iter[200] loss: [0.3220807611942291, 1.70854914188385] \n",
      "epoch[0] iter[240] loss: [0.22905227541923523, 1.1642730236053467] \n",
      "epoch[0] iter[280] loss: [0.20727740228176117, 1.098405361175537] \n",
      "epoch[0] iter[320] loss: [0.2574497163295746, 1.2304084300994873] \n",
      "epoch[0] iter[360] loss: [0.19968612492084503, 0.9792288541793823] \n",
      "epoch[0] iter[400] loss: [0.18470756709575653, 0.8610690832138062] \n",
      "epoch[0] iter[440] loss: [0.20014259219169617, 1.190737009048462] \n",
      "epoch[0] iter[480] loss: [0.2056359350681305, 1.1250542402267456] \n",
      "epoch[0] iter[520] loss: [0.19921275973320007, 1.0406765937805176] \n",
      "epoch[0] iter[560] loss: [0.16296595335006714, 0.7250649929046631] \n",
      "epoch[0] iter[600] loss: [0.1731620877981186, 0.7727018594741821] \n",
      "epoch[0] iter[640] loss: [0.1727517992258072, 0.7469112277030945] \n",
      "epoch[0] iter[680] loss: [0.1692916452884674, 0.738254964351654] \n",
      "epoch[0] iter[720] loss: [0.1641855239868164, 0.6856439113616943] \n",
      "epoch[0] iter[760] loss: [0.19593340158462524, 0.8850458860397339] \n",
      "Finish epoch[0], time elapsed[1068.0136308670044]\n",
      "epoch[1] iter[0] loss: [0.17167258262634277, 0.8580548167228699] \n",
      "epoch[1] iter[40] loss: [0.14620940387248993, 0.5189585089683533] \n",
      "epoch[1] iter[80] loss: [0.16066887974739075, 0.6086102724075317] \n",
      "epoch[1] iter[120] loss: [0.1505560278892517, 0.4376538097858429] \n",
      "epoch[1] iter[160] loss: [0.19049298763275146, 0.582294762134552] \n",
      "epoch[1] iter[200] loss: [0.1850634217262268, 0.5230676531791687] \n",
      "epoch[1] iter[240] loss: [0.16458795964717865, 0.3984394073486328] \n",
      "epoch[1] iter[280] loss: [0.16351579129695892, 0.4054163098335266] \n",
      "epoch[1] iter[320] loss: [0.15438202023506165, 0.5407759547233582] \n",
      "epoch[1] iter[360] loss: [0.15689732134342194, 0.6229200959205627] \n",
      "epoch[1] iter[400] loss: [0.13296137750148773, 0.5052148103713989] \n",
      "epoch[1] iter[440] loss: [0.15155155956745148, 0.2777084708213806] \n",
      "epoch[1] iter[480] loss: [0.144389808177948, 0.3074263036251068] \n",
      "epoch[1] iter[520] loss: [0.13966906070709229, 0.43758687376976013] \n",
      "epoch[1] iter[560] loss: [0.14683212339878082, 0.4061702489852905] \n",
      "epoch[1] iter[600] loss: [0.13025914132595062, 0.2616182863712311] \n",
      "epoch[1] iter[640] loss: [0.17180907726287842, 0.5969685912132263] \n",
      "epoch[1] iter[680] loss: [0.12626110017299652, 0.24008232355117798] \n",
      "epoch[1] iter[720] loss: [0.13014529645442963, 0.212619811296463] \n",
      "epoch[1] iter[760] loss: [0.13962042331695557, 0.2472878098487854] \n",
      "Finish epoch[1], time elapsed[567.1027059555054]\n",
      "epoch[2] iter[0] loss: [0.16548950970172882, 0.3076675534248352] \n",
      "epoch[2] iter[40] loss: [0.11324194818735123, 0.15466013550758362] \n",
      "epoch[2] iter[80] loss: [0.14703011512756348, 0.23286233842372894] \n",
      "epoch[2] iter[120] loss: [0.14061339199543, 0.35655325651168823] \n",
      "epoch[2] iter[160] loss: [0.1536083221435547, 0.21000473201274872] \n",
      "epoch[2] iter[200] loss: [0.13124172389507294, 0.22059299051761627] \n",
      "epoch[2] iter[240] loss: [0.13180401921272278, 0.1779610514640808] \n",
      "epoch[2] iter[280] loss: [0.15637309849262238, 0.331978440284729] \n",
      "epoch[2] iter[320] loss: [0.114992655813694, 0.21160291135311127] \n",
      "epoch[2] iter[360] loss: [0.12133300304412842, 0.2893817126750946] \n",
      "epoch[2] iter[400] loss: [0.14650090038776398, 0.2492096722126007] \n",
      "epoch[2] iter[440] loss: [0.1167469471693039, 0.1345631182193756] \n",
      "epoch[2] iter[480] loss: [0.18844027817249298, 0.284109890460968] \n",
      "epoch[2] iter[520] loss: [0.14130671322345734, 0.1626313179731369] \n",
      "epoch[2] iter[560] loss: [0.14178244769573212, 0.16744491457939148] \n",
      "epoch[2] iter[600] loss: [0.13143937289714813, 0.2112589031457901] \n",
      "epoch[2] iter[640] loss: [0.13119545578956604, 0.1540287286043167] \n",
      "epoch[2] iter[680] loss: [0.11719667911529541, 0.14351162314414978] \n",
      "epoch[2] iter[720] loss: [0.14412572979927063, 0.1714833527803421] \n",
      "epoch[2] iter[760] loss: [0.13414143025875092, 0.1918378323316574] \n",
      "Finish epoch[2], time elapsed[565.5313251018524]\n",
      "epoch[3] iter[0] loss: [0.13660767674446106, 0.3275320827960968] \n",
      "epoch[3] iter[40] loss: [0.12923520803451538, 0.31102806329727173] \n",
      "epoch[3] iter[80] loss: [0.11728992313146591, 0.26066863536834717] \n",
      "epoch[3] iter[120] loss: [0.10688882321119308, 0.15058204531669617] \n",
      "epoch[3] iter[160] loss: [0.13318921625614166, 0.19279970228672028] \n",
      "epoch[3] iter[200] loss: [0.1487988829612732, 0.2113538384437561] \n",
      "epoch[3] iter[240] loss: [0.15285466611385345, 0.14940237998962402] \n",
      "epoch[3] iter[280] loss: [0.1275397390127182, 0.197297602891922] \n",
      "epoch[3] iter[320] loss: [0.16438379883766174, 0.22522898018360138] \n",
      "epoch[3] iter[360] loss: [0.1233072578907013, 0.11502705514431] \n",
      "epoch[3] iter[400] loss: [0.1110258623957634, 0.1638239324092865] \n",
      "epoch[3] iter[440] loss: [0.12801389396190643, 0.1522780805826187] \n",
      "epoch[3] iter[480] loss: [0.11806724220514297, 0.13345560431480408] \n",
      "epoch[3] iter[520] loss: [0.11707261949777603, 0.1555897742509842] \n",
      "epoch[3] iter[560] loss: [0.1483449786901474, 0.17849348485469818] \n",
      "epoch[3] iter[600] loss: [0.12688203155994415, 0.22739188373088837] \n",
      "epoch[3] iter[640] loss: [0.112594835460186, 0.14874067902565002] \n",
      "epoch[3] iter[680] loss: [0.12873311340808868, 0.20739197731018066] \n",
      "epoch[3] iter[720] loss: [0.14445364475250244, 0.18561646342277527] \n",
      "epoch[3] iter[760] loss: [0.1525586098432541, 0.18726365268230438] \n",
      "Finish epoch[3], time elapsed[566.9753336906433]\n",
      "epoch[4] iter[0] loss: [0.1432805359363556, 0.20370018482208252] \n",
      "epoch[4] iter[40] loss: [0.13001632690429688, 0.15429136157035828] \n",
      "epoch[4] iter[80] loss: [0.10074973106384277, 0.11492955684661865] \n",
      "epoch[4] iter[120] loss: [0.11263963580131531, 0.13041721284389496] \n",
      "epoch[4] iter[160] loss: [0.15856607258319855, 0.22131814062595367] \n",
      "epoch[4] iter[200] loss: [0.12681886553764343, 0.1728888750076294] \n",
      "epoch[4] iter[240] loss: [0.13618838787078857, 0.23597879707813263] \n",
      "epoch[4] iter[280] loss: [0.11687424778938293, 0.12989601492881775] \n",
      "epoch[4] iter[320] loss: [0.1105925515294075, 0.12819291651248932] \n",
      "epoch[4] iter[360] loss: [0.14437447488307953, 0.13874779641628265] \n",
      "epoch[4] iter[400] loss: [0.13363298773765564, 0.16444441676139832] \n",
      "epoch[4] iter[440] loss: [0.119931161403656, 0.13474373519420624] \n",
      "epoch[4] iter[480] loss: [0.12050402909517288, 0.1370222568511963] \n",
      "epoch[4] iter[520] loss: [0.13281852006912231, 0.1387844681739807] \n",
      "epoch[4] iter[560] loss: [0.09980062395334244, 0.11615785211324692] \n",
      "epoch[4] iter[600] loss: [0.12950898706912994, 0.11578576266765594] \n",
      "epoch[4] iter[640] loss: [0.13155533373355865, 0.17151540517807007] \n",
      "epoch[4] iter[680] loss: [0.138838529586792, 0.12899892032146454] \n",
      "epoch[4] iter[720] loss: [0.13950186967849731, 0.15894582867622375] \n",
      "epoch[4] iter[760] loss: [0.1254083216190338, 0.13503211736679077] \n",
      "Finish epoch[4], time elapsed[573.4371223449707]\n",
      "epoch[5] iter[0] loss: [0.11246166378259659, 0.14132988452911377] \n",
      "epoch[5] iter[40] loss: [0.10943267494440079, 0.10547985136508942] \n",
      "epoch[5] iter[80] loss: [0.15641382336616516, 0.1697247326374054] \n",
      "epoch[5] iter[120] loss: [0.12035281211137772, 0.12291846424341202] \n",
      "epoch[5] iter[160] loss: [0.11412211507558823, 0.14272528886795044] \n",
      "epoch[5] iter[200] loss: [0.1154438778758049, 0.14466004073619843] \n",
      "epoch[5] iter[240] loss: [0.1113056093454361, 0.12394627183675766] \n",
      "epoch[5] iter[280] loss: [0.12308286130428314, 0.1745951771736145] \n",
      "epoch[5] iter[320] loss: [0.14529484510421753, 0.18854542076587677] \n",
      "epoch[5] iter[360] loss: [0.11073607951402664, 0.09384187310934067] \n",
      "epoch[5] iter[400] loss: [0.11959834396839142, 0.1280658096075058] \n",
      "epoch[5] iter[440] loss: [0.12784892320632935, 0.11683794856071472] \n",
      "epoch[5] iter[480] loss: [0.10750756412744522, 0.10563825070858002] \n",
      "epoch[5] iter[520] loss: [0.0995771661400795, 0.14566059410572052] \n",
      "epoch[5] iter[560] loss: [0.10863432288169861, 0.129347562789917] \n",
      "epoch[5] iter[600] loss: [0.10689828544855118, 0.13564518094062805] \n",
      "epoch[5] iter[640] loss: [0.11302850395441055, 0.11259976029396057] \n",
      "epoch[5] iter[680] loss: [0.1171058863401413, 0.14119155704975128] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch[5] iter[720] loss: [0.10072840750217438, 0.13493019342422485] \n",
      "epoch[5] iter[760] loss: [0.13404706120491028, 0.18737109005451202] \n",
      "Finish epoch[5], time elapsed[573.5729403495789]\n",
      "epoch[6] iter[0] loss: [0.10672300308942795, 0.1267934888601303] \n",
      "epoch[6] iter[40] loss: [0.12119714915752411, 0.16922158002853394] \n",
      "epoch[6] iter[80] loss: [0.1202646866440773, 0.10948043316602707] \n",
      "epoch[6] iter[120] loss: [0.12247998267412186, 0.1384057253599167] \n",
      "epoch[6] iter[160] loss: [0.109781414270401, 0.1253279149532318] \n",
      "epoch[6] iter[200] loss: [0.1245444193482399, 0.14087429642677307] \n",
      "epoch[6] iter[240] loss: [0.12204711139202118, 0.12788045406341553] \n",
      "epoch[6] iter[280] loss: [0.12381856143474579, 0.11439073830842972] \n",
      "epoch[6] iter[320] loss: [0.12689024209976196, 0.13593588769435883] \n",
      "epoch[6] iter[360] loss: [0.12371483445167542, 0.27814316749572754] \n",
      "epoch[6] iter[400] loss: [0.12218965590000153, 0.12819623947143555] \n",
      "epoch[6] iter[440] loss: [0.14200244843959808, 0.1761951744556427] \n",
      "epoch[6] iter[480] loss: [0.11890221387147903, 0.14059637486934662] \n",
      "epoch[6] iter[520] loss: [0.10311712324619293, 0.12927334010601044] \n",
      "epoch[6] iter[560] loss: [0.10907911509275436, 0.18846024572849274] \n",
      "epoch[6] iter[600] loss: [0.123682402074337, 0.13714618980884552] \n",
      "epoch[6] iter[640] loss: [0.13156555593013763, 0.12815752625465393] \n",
      "epoch[6] iter[680] loss: [0.13614223897457123, 0.1268453299999237] \n",
      "epoch[6] iter[720] loss: [0.10996469110250473, 0.14940285682678223] \n",
      "epoch[6] iter[760] loss: [0.10683757811784744, 0.09670600295066833] \n",
      "Finish epoch[6], time elapsed[569.8184978961945]\n",
      "epoch[7] iter[0] loss: [0.11801505088806152, 0.1078500971198082] \n",
      "epoch[7] iter[40] loss: [0.11008891463279724, 0.2076665163040161] \n",
      "epoch[7] iter[80] loss: [0.11898097395896912, 0.1374550312757492] \n",
      "epoch[7] iter[120] loss: [0.09870506078004837, 0.08792984485626221] \n",
      "epoch[7] iter[160] loss: [0.11526288092136383, 0.17453435063362122] \n",
      "epoch[7] iter[200] loss: [0.1329721361398697, 0.2138754278421402] \n",
      "epoch[7] iter[240] loss: [0.08596590161323547, 0.08741720765829086] \n",
      "epoch[7] iter[280] loss: [0.12482479959726334, 0.13383662700653076] \n",
      "epoch[7] iter[320] loss: [0.10356416553258896, 0.13033775985240936] \n",
      "epoch[7] iter[360] loss: [0.10767562687397003, 0.1415661722421646] \n",
      "epoch[7] iter[400] loss: [0.1165061667561531, 0.1301073282957077] \n",
      "epoch[7] iter[440] loss: [0.11822223663330078, 0.17241860926151276] \n",
      "epoch[7] iter[480] loss: [0.12025321274995804, 0.12590044736862183] \n",
      "epoch[7] iter[520] loss: [0.12116236239671707, 0.1405530571937561] \n",
      "epoch[7] iter[560] loss: [0.12272074073553085, 0.10764668881893158] \n",
      "epoch[7] iter[600] loss: [0.14652958512306213, 0.22833919525146484] \n",
      "epoch[7] iter[640] loss: [0.11109975725412369, 0.11454558372497559] \n",
      "epoch[7] iter[680] loss: [0.12006638199090958, 0.11378508061170578] \n",
      "epoch[7] iter[720] loss: [0.10882548242807388, 0.09979799389839172] \n",
      "epoch[7] iter[760] loss: [0.11499688774347305, 0.11990368366241455] \n",
      "Finish epoch[7], time elapsed[570.9821057319641]\n",
      "epoch[8] iter[0] loss: [0.1269797682762146, 0.14618922770023346] \n",
      "epoch[8] iter[40] loss: [0.09790600836277008, 0.09727533161640167] \n",
      "epoch[8] iter[80] loss: [0.16818252205848694, 0.1887950897216797] \n",
      "epoch[8] iter[120] loss: [0.13349390029907227, 0.12481042742729187] \n",
      "epoch[8] iter[160] loss: [0.12133271247148514, 0.16137048602104187] \n",
      "epoch[8] iter[200] loss: [0.11157088726758957, 0.13340424001216888] \n",
      "epoch[8] iter[240] loss: [0.12149598449468613, 0.1881706714630127] \n",
      "epoch[8] iter[280] loss: [0.10234151035547256, 0.12264175713062286] \n",
      "epoch[8] iter[320] loss: [0.12653447687625885, 0.12634791433811188] \n",
      "epoch[8] iter[360] loss: [0.09769521653652191, 0.12960697710514069] \n",
      "epoch[8] iter[400] loss: [0.1397852897644043, 0.14647231996059418] \n",
      "epoch[8] iter[440] loss: [0.11170811951160431, 0.09488912671804428] \n",
      "epoch[8] iter[480] loss: [0.10413850098848343, 0.1121966540813446] \n",
      "epoch[8] iter[520] loss: [0.12140936404466629, 0.12327918410301208] \n",
      "epoch[8] iter[560] loss: [0.11102268099784851, 0.11456450074911118] \n",
      "epoch[8] iter[600] loss: [0.097632497549057, 0.08372432738542557] \n",
      "epoch[8] iter[640] loss: [0.10994210094213486, 0.10243424028158188] \n",
      "epoch[8] iter[680] loss: [0.11233226954936981, 0.12120228260755539] \n",
      "epoch[8] iter[720] loss: [0.11756984144449234, 0.14219626784324646] \n",
      "epoch[8] iter[760] loss: [0.11310341209173203, 0.11272195726633072] \n",
      "Finish epoch[8], time elapsed[568.6066188812256]\n",
      "epoch[9] iter[0] loss: [0.11488572508096695, 0.09944722801446915] \n",
      "epoch[9] iter[40] loss: [0.10305717587471008, 0.09779659658670425] \n",
      "epoch[9] iter[80] loss: [0.1046152263879776, 0.09168431907892227] \n",
      "epoch[9] iter[120] loss: [0.1014256402850151, 0.09132357686758041] \n",
      "epoch[9] iter[160] loss: [0.10141339898109436, 0.11463305354118347] \n",
      "epoch[9] iter[200] loss: [0.11138034611940384, 0.14592593908309937] \n",
      "epoch[9] iter[240] loss: [0.11424203962087631, 0.10725020617246628] \n",
      "epoch[9] iter[280] loss: [0.10968638211488724, 0.11369866132736206] \n",
      "epoch[9] iter[320] loss: [0.09688788652420044, 0.1063087061047554] \n",
      "epoch[9] iter[360] loss: [0.11028625816106796, 0.11019919812679291] \n",
      "epoch[9] iter[400] loss: [0.10859604179859161, 0.10141164809465408] \n",
      "epoch[9] iter[440] loss: [0.11026076972484589, 0.09725631028413773] \n",
      "epoch[9] iter[480] loss: [0.10691660642623901, 0.09980113804340363] \n",
      "epoch[9] iter[520] loss: [0.09294102340936661, 0.09442582726478577] \n",
      "epoch[9] iter[560] loss: [0.10937588661909103, 0.18703970313072205] \n",
      "epoch[9] iter[600] loss: [0.11078574508428574, 0.13304679095745087] \n",
      "epoch[9] iter[640] loss: [0.13925796747207642, 0.225339874625206] \n",
      "epoch[9] iter[680] loss: [0.11465869098901749, 0.10792627185583115] \n",
      "epoch[9] iter[720] loss: [0.10922703146934509, 0.09420149773359299] \n",
      "epoch[9] iter[760] loss: [0.1120271384716034, 0.10514894127845764] \n",
      "Finish epoch[9], time elapsed[570.913557767868]\n",
      "epoch[10] iter[0] loss: [0.09824461489915848, 0.11425076425075531] \n",
      "epoch[10] iter[40] loss: [0.10114841908216476, 0.10624933987855911] \n",
      "epoch[10] iter[80] loss: [0.1246919333934784, 0.15534734725952148] \n",
      "epoch[10] iter[120] loss: [0.10701745748519897, 0.10547718405723572] \n",
      "epoch[10] iter[160] loss: [0.14626409113407135, 0.1463390588760376] \n",
      "epoch[10] iter[200] loss: [0.10589559376239777, 0.09053201228380203] \n",
      "epoch[10] iter[240] loss: [0.09638024121522903, 0.09306387603282928] \n",
      "epoch[10] iter[280] loss: [0.09097972512245178, 0.0751589834690094] \n",
      "epoch[10] iter[320] loss: [0.10238170623779297, 0.12206081300973892] \n",
      "epoch[10] iter[360] loss: [0.2050604373216629, 0.2537614405155182] \n",
      "epoch[10] iter[400] loss: [0.11661966145038605, 0.14554700255393982] \n",
      "epoch[10] iter[440] loss: [0.1023774966597557, 0.09255613386631012] \n",
      "epoch[10] iter[480] loss: [0.10897474735975266, 0.10005715489387512] \n",
      "epoch[10] iter[520] loss: [0.09213648736476898, 0.08512170612812042] \n",
      "epoch[10] iter[560] loss: [0.11443335562944412, 0.1154814213514328] \n",
      "epoch[10] iter[600] loss: [0.10454624891281128, 0.1068548858165741] \n",
      "epoch[10] iter[640] loss: [0.09713168442249298, 0.09304150938987732] \n",
      "epoch[10] iter[680] loss: [0.10069718956947327, 0.08646377921104431] \n",
      "epoch[10] iter[720] loss: [0.10339030623435974, 0.11155947297811508] \n",
      "epoch[10] iter[760] loss: [0.10692182928323746, 0.11563599109649658] \n",
      "Finish epoch[10], time elapsed[568.6947977542877]\n",
      "epoch[11] iter[0] loss: [0.10150039941072464, 0.11396470665931702] \n",
      "epoch[11] iter[40] loss: [0.08307508379220963, 0.08401337265968323] \n",
      "epoch[11] iter[80] loss: [0.106224924325943, 0.09829876571893692] \n",
      "epoch[11] iter[120] loss: [0.10004693269729614, 0.11370932310819626] \n",
      "epoch[11] iter[160] loss: [0.10456489771604538, 0.09927360713481903] \n",
      "epoch[11] iter[200] loss: [0.09753817319869995, 0.11316974461078644] \n",
      "epoch[11] iter[240] loss: [0.11069659143686295, 0.12524858117103577] \n",
      "epoch[11] iter[280] loss: [0.1040821447968483, 0.09073276817798615] \n",
      "epoch[11] iter[320] loss: [0.1109805554151535, 0.1015610545873642] \n",
      "epoch[11] iter[360] loss: [0.11602159589529037, 0.1182083785533905] \n",
      "epoch[11] iter[400] loss: [0.08833056688308716, 0.07904665172100067] \n",
      "epoch[11] iter[440] loss: [0.09818308800458908, 0.09385824203491211] \n",
      "epoch[11] iter[480] loss: [0.09415414184331894, 0.08363280445337296] \n",
      "epoch[11] iter[520] loss: [0.11095447093248367, 0.10145555436611176] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch[11] iter[560] loss: [0.1030280590057373, 0.10059009492397308] \n",
      "epoch[11] iter[600] loss: [0.12584616243839264, 0.125372514128685] \n",
      "epoch[11] iter[640] loss: [0.1106935515999794, 0.11624641716480255] \n",
      "epoch[11] iter[680] loss: [0.10660260170698166, 0.1745986044406891] \n",
      "epoch[11] iter[720] loss: [0.10348568856716156, 0.09063015878200531] \n",
      "epoch[11] iter[760] loss: [0.1048506423830986, 0.08312682807445526] \n",
      "Finish epoch[11], time elapsed[566.0226566791534]\n",
      "epoch[12] iter[0] loss: [0.11142316460609436, 0.09651367366313934] \n",
      "epoch[12] iter[40] loss: [0.10900110751390457, 0.09071719646453857] \n",
      "epoch[12] iter[80] loss: [0.10442588478326797, 0.0905236229300499] \n",
      "epoch[12] iter[120] loss: [0.10256490856409073, 0.12016136944293976] \n",
      "epoch[12] iter[160] loss: [0.1296527236700058, 0.14576919376850128] \n",
      "epoch[12] iter[200] loss: [0.11970382928848267, 0.1460089236497879] \n",
      "epoch[12] iter[240] loss: [0.0893872007727623, 0.08093293011188507] \n",
      "epoch[12] iter[280] loss: [0.10912200808525085, 0.08051547408103943] \n",
      "epoch[12] iter[320] loss: [0.11066607385873795, 0.10983404517173767] \n",
      "epoch[12] iter[360] loss: [0.09920351207256317, 0.07624978572130203] \n",
      "epoch[12] iter[400] loss: [0.096488356590271, 0.12621626257896423] \n",
      "epoch[12] iter[440] loss: [0.09372846782207489, 0.1117313951253891] \n",
      "epoch[12] iter[480] loss: [0.09159383922815323, 0.0759120061993599] \n",
      "epoch[12] iter[520] loss: [0.09947507083415985, 0.10678021609783173] \n",
      "epoch[12] iter[560] loss: [0.09214815497398376, 0.08131527900695801] \n",
      "epoch[12] iter[600] loss: [0.10987286269664764, 0.09570401906967163] \n",
      "epoch[12] iter[640] loss: [0.10233084857463837, 0.0894426703453064] \n",
      "epoch[12] iter[680] loss: [0.08932600915431976, 0.08773207664489746] \n",
      "epoch[12] iter[720] loss: [0.10210592299699783, 0.07947997748851776] \n",
      "epoch[12] iter[760] loss: [0.11048055440187454, 0.12204860150814056] \n",
      "Finish epoch[12], time elapsed[568.7044401168823]\n",
      "epoch[13] iter[0] loss: [0.10831896215677261, 0.10264156013727188] \n",
      "epoch[13] iter[40] loss: [0.11043937504291534, 0.12674680352210999] \n",
      "epoch[13] iter[80] loss: [0.1231556087732315, 0.11612477898597717] \n",
      "epoch[13] iter[120] loss: [0.10702961683273315, 0.1036229133605957] \n",
      "epoch[13] iter[160] loss: [0.10154372453689575, 0.11217888444662094] \n",
      "epoch[13] iter[200] loss: [0.12187200039625168, 0.10898387432098389] \n",
      "epoch[13] iter[240] loss: [0.12298101931810379, 0.10353416204452515] \n",
      "epoch[13] iter[280] loss: [0.09540647268295288, 0.09293316304683685] \n",
      "epoch[13] iter[320] loss: [0.11582695692777634, 0.15749914944171906] \n",
      "epoch[13] iter[360] loss: [0.10390639305114746, 0.12756073474884033] \n",
      "epoch[13] iter[400] loss: [0.10263389348983765, 0.12666314840316772] \n",
      "epoch[13] iter[440] loss: [0.10270029306411743, 0.0998176783323288] \n",
      "epoch[13] iter[480] loss: [0.09560374170541763, 0.0925455391407013] \n",
      "epoch[13] iter[520] loss: [0.09584712982177734, 0.09864833950996399] \n",
      "epoch[13] iter[560] loss: [0.09898293763399124, 0.09041177481412888] \n",
      "epoch[13] iter[600] loss: [0.1059495285153389, 0.20131295919418335] \n",
      "epoch[13] iter[640] loss: [0.12415006756782532, 0.10172705352306366] \n",
      "epoch[13] iter[680] loss: [0.1075955256819725, 0.11104341596364975] \n",
      "epoch[13] iter[720] loss: [0.11361303180456161, 0.10735247284173965] \n",
      "epoch[13] iter[760] loss: [0.11310917884111404, 0.09400352835655212] \n",
      "Finish epoch[13], time elapsed[566.2437636852264]\n",
      "epoch[14] iter[0] loss: [0.11437671631574631, 0.11161157488822937] \n",
      "epoch[14] iter[40] loss: [0.11195167899131775, 0.10158760100603104] \n",
      "epoch[14] iter[80] loss: [0.09677967429161072, 0.08759143203496933] \n",
      "epoch[14] iter[120] loss: [0.0894065871834755, 0.08649516105651855] \n",
      "epoch[14] iter[160] loss: [0.10709647089242935, 0.08799798786640167] \n",
      "epoch[14] iter[200] loss: [0.09487542510032654, 0.10041088610887527] \n",
      "epoch[14] iter[240] loss: [0.11657193303108215, 0.0983121395111084] \n",
      "epoch[14] iter[280] loss: [0.10939781367778778, 0.09647995978593826] \n",
      "epoch[14] iter[320] loss: [0.10291764885187149, 0.09501687437295914] \n",
      "epoch[14] iter[360] loss: [0.10576468706130981, 0.1190129742026329] \n",
      "epoch[14] iter[400] loss: [0.12013685703277588, 0.10230018943548203] \n",
      "epoch[14] iter[440] loss: [0.13669145107269287, 0.158721923828125] \n",
      "epoch[14] iter[480] loss: [0.10308471322059631, 0.08564795553684235] \n",
      "epoch[14] iter[520] loss: [0.09712214022874832, 0.09105902165174484] \n",
      "epoch[14] iter[560] loss: [0.08712258189916611, 0.08176694810390472] \n",
      "epoch[14] iter[600] loss: [0.09060623496770859, 0.08173765242099762] \n",
      "epoch[14] iter[640] loss: [0.10119060426950455, 0.10080571472644806] \n",
      "epoch[14] iter[680] loss: [0.09892834722995758, 0.12297786772251129] \n",
      "epoch[14] iter[720] loss: [0.10786642879247665, 0.11277181655168533] \n",
      "epoch[14] iter[760] loss: [0.1254279911518097, 0.12157592177391052] \n",
      "Finish epoch[14], time elapsed[564.1379065513611]\n",
      "epoch[15] iter[0] loss: [0.1066850870847702, 0.09255492687225342] \n",
      "epoch[15] iter[40] loss: [0.10149572789669037, 0.0976657047867775] \n",
      "epoch[15] iter[80] loss: [0.1077006608247757, 0.1255951225757599] \n",
      "epoch[15] iter[120] loss: [0.11000479012727737, 0.09416060149669647] \n",
      "epoch[15] iter[160] loss: [0.09418008476495743, 0.07637163251638412] \n",
      "epoch[15] iter[200] loss: [0.1310471147298813, 0.14295396208763123] \n",
      "epoch[15] iter[240] loss: [0.08915191143751144, 0.14798679947853088] \n",
      "epoch[15] iter[280] loss: [0.09881309419870377, 0.1436789631843567] \n",
      "epoch[15] iter[320] loss: [0.12908688187599182, 0.1385064274072647] \n",
      "epoch[15] iter[360] loss: [0.09463783353567123, 0.07441987097263336] \n",
      "epoch[15] iter[400] loss: [0.08984021097421646, 0.07064434885978699] \n",
      "epoch[15] iter[440] loss: [0.10673156380653381, 0.10923617333173752] \n",
      "epoch[15] iter[480] loss: [0.09663316607475281, 0.09233979880809784] \n",
      "epoch[15] iter[520] loss: [0.1115703359246254, 0.11156851053237915] \n",
      "epoch[15] iter[560] loss: [0.10774517059326172, 0.09202592074871063] \n",
      "epoch[15] iter[600] loss: [0.10101522505283356, 0.08768431097269058] \n",
      "epoch[15] iter[640] loss: [0.11611445248126984, 0.09826882928609848] \n",
      "epoch[15] iter[680] loss: [0.09968104213476181, 0.0909075140953064] \n",
      "epoch[15] iter[720] loss: [0.10340873897075653, 0.08888616412878036] \n",
      "epoch[15] iter[760] loss: [0.08871868997812271, 0.14702340960502625] \n",
      "Finish epoch[15], time elapsed[565.6931364536285]\n",
      "epoch[16] iter[0] loss: [0.09482157975435257, 0.08098926395177841] \n",
      "epoch[16] iter[40] loss: [0.09810031950473785, 0.09414015710353851] \n",
      "epoch[16] iter[80] loss: [0.09958182275295258, 0.09439964592456818] \n",
      "epoch[16] iter[120] loss: [0.08991000801324844, 0.1054762527346611] \n",
      "epoch[16] iter[160] loss: [0.08242114633321762, 0.06626930087804794] \n",
      "epoch[16] iter[200] loss: [0.10756129026412964, 0.10642381012439728] \n",
      "epoch[16] iter[240] loss: [0.10103815793991089, 0.11767590045928955] \n",
      "epoch[16] iter[280] loss: [0.1427883356809616, 0.15400062501430511] \n",
      "epoch[16] iter[320] loss: [0.10521256178617477, 0.11415073275566101] \n",
      "epoch[16] iter[360] loss: [0.1087215468287468, 0.10561304539442062] \n",
      "epoch[16] iter[400] loss: [0.1052602082490921, 0.10582850128412247] \n",
      "epoch[16] iter[440] loss: [0.10950370877981186, 0.09697769582271576] \n",
      "epoch[16] iter[480] loss: [0.11468290537595749, 0.11186306178569794] \n",
      "epoch[16] iter[520] loss: [0.1047692745923996, 0.09949809312820435] \n",
      "epoch[16] iter[560] loss: [0.1076493114233017, 0.10413853079080582] \n",
      "epoch[16] iter[600] loss: [0.1095702275633812, 0.1259835809469223] \n",
      "epoch[16] iter[640] loss: [0.11146518588066101, 0.10649462044239044] \n",
      "epoch[16] iter[680] loss: [0.08991942554712296, 0.09307754039764404] \n",
      "epoch[16] iter[720] loss: [0.1017836481332779, 0.11062688380479813] \n",
      "epoch[16] iter[760] loss: [0.09363225102424622, 0.09689150005578995] \n",
      "Finish epoch[16], time elapsed[566.2559986114502]\n",
      "epoch[17] iter[0] loss: [0.1099301278591156, 0.1054263710975647] \n",
      "epoch[17] iter[40] loss: [0.10852722078561783, 0.07719141244888306] \n",
      "epoch[17] iter[80] loss: [0.08688358962535858, 0.0694945678114891] \n",
      "epoch[17] iter[120] loss: [0.0884445458650589, 0.07859893143177032] \n",
      "epoch[17] iter[160] loss: [0.08331593126058578, 0.06343097239732742] \n",
      "epoch[17] iter[200] loss: [0.09553252160549164, 0.09379761666059494] \n",
      "epoch[17] iter[240] loss: [0.1212308257818222, 0.10076164454221725] \n",
      "epoch[17] iter[280] loss: [0.08904055505990982, 0.08628995716571808] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch[17] iter[320] loss: [0.10383665561676025, 0.08533774316310883] \n",
      "epoch[17] iter[360] loss: [0.08769770711660385, 0.08349085599184036] \n",
      "epoch[17] iter[400] loss: [0.09543406963348389, 0.0819442868232727] \n",
      "epoch[17] iter[440] loss: [0.09007903188467026, 0.09977488964796066] \n",
      "epoch[17] iter[480] loss: [0.1157650575041771, 0.10148242861032486] \n",
      "epoch[17] iter[520] loss: [0.10331682115793228, 0.08541423082351685] \n",
      "epoch[17] iter[560] loss: [0.09712280333042145, 0.0896749198436737] \n",
      "epoch[17] iter[600] loss: [0.10210300981998444, 0.08445873111486435] \n",
      "epoch[17] iter[640] loss: [0.09493560343980789, 0.08734259754419327] \n",
      "epoch[17] iter[680] loss: [0.10790259391069412, 0.12114717066287994] \n",
      "epoch[17] iter[720] loss: [0.10348819196224213, 0.07659769058227539] \n",
      "epoch[17] iter[760] loss: [0.1355619728565216, 0.16952164471149445] \n",
      "Finish epoch[17], time elapsed[565.1305546760559]\n",
      "epoch[18] iter[0] loss: [0.09320273250341415, 0.07198861241340637] \n",
      "epoch[18] iter[40] loss: [0.08883789926767349, 0.07009199261665344] \n",
      "epoch[18] iter[80] loss: [0.10592514276504517, 0.09826130419969559] \n",
      "epoch[18] iter[120] loss: [0.09274569898843765, 0.08700613677501678] \n",
      "epoch[18] iter[160] loss: [0.09472507238388062, 0.06936903297901154] \n",
      "epoch[18] iter[200] loss: [0.08893883228302002, 0.07928138226270676] \n",
      "epoch[18] iter[240] loss: [0.09345881640911102, 0.0767071396112442] \n",
      "epoch[18] iter[280] loss: [0.12120456248521805, 0.10184825956821442] \n",
      "epoch[18] iter[320] loss: [0.09626822173595428, 0.08367114514112473] \n",
      "epoch[18] iter[360] loss: [0.10947578400373459, 0.15093426406383514] \n",
      "epoch[18] iter[400] loss: [0.0941232293844223, 0.08271162956953049] \n",
      "epoch[18] iter[440] loss: [0.10629706084728241, 0.10627268254756927] \n",
      "epoch[18] iter[480] loss: [0.09923256933689117, 0.09227859228849411] \n",
      "epoch[18] iter[520] loss: [0.09449367225170135, 0.07484303414821625] \n",
      "epoch[18] iter[560] loss: [0.0867413803935051, 0.08066875487565994] \n",
      "epoch[18] iter[600] loss: [0.1195434182882309, 0.0890236422419548] \n",
      "epoch[18] iter[640] loss: [0.10219414532184601, 0.0733124315738678] \n",
      "epoch[18] iter[680] loss: [0.09504998475313187, 0.07804819941520691] \n",
      "epoch[18] iter[720] loss: [0.12326949834823608, 0.19485649466514587] \n",
      "epoch[18] iter[760] loss: [0.10001261532306671, 0.10781776160001755] \n",
      "Finish epoch[18], time elapsed[566.7951264381409]\n",
      "epoch[19] iter[0] loss: [0.0875917449593544, 0.07736815512180328] \n",
      "epoch[19] iter[40] loss: [0.08692245185375214, 0.0690881758928299] \n",
      "epoch[19] iter[80] loss: [0.09159478545188904, 0.07524057477712631] \n",
      "epoch[19] iter[120] loss: [0.11338475346565247, 0.10917530953884125] \n",
      "epoch[19] iter[160] loss: [0.09312871098518372, 0.08578188717365265] \n",
      "epoch[19] iter[200] loss: [0.09764920175075531, 0.12198970466852188] \n",
      "epoch[19] iter[240] loss: [0.1105460524559021, 0.08855810016393661] \n",
      "epoch[19] iter[280] loss: [0.09309091418981552, 0.07328397035598755] \n",
      "epoch[19] iter[320] loss: [0.09532620757818222, 0.09614179283380508] \n",
      "epoch[19] iter[360] loss: [0.12511146068572998, 0.13797953724861145] \n",
      "epoch[19] iter[400] loss: [0.09302786737680435, 0.09558062255382538] \n",
      "epoch[19] iter[440] loss: [0.10205139219760895, 0.10951309651136398] \n",
      "epoch[19] iter[480] loss: [0.1292366087436676, 0.17157115042209625] \n",
      "epoch[19] iter[520] loss: [0.08621080964803696, 0.12200453877449036] \n",
      "epoch[19] iter[560] loss: [0.1289537250995636, 0.1145702674984932] \n",
      "epoch[19] iter[600] loss: [0.09679102152585983, 0.09357298910617828] \n",
      "epoch[19] iter[640] loss: [0.09380297362804413, 0.08344893902540207] \n",
      "epoch[19] iter[680] loss: [0.11948180198669434, 0.12012635916471481] \n",
      "epoch[19] iter[720] loss: [0.08810734003782272, 0.11202548444271088] \n",
      "epoch[19] iter[760] loss: [0.1013731136918068, 0.09925968945026398] \n",
      "Finish epoch[19], time elapsed[571.4846169948578]\n",
      "epoch[20] iter[0] loss: [0.11135223507881165, 0.09370098263025284] \n",
      "epoch[20] iter[40] loss: [0.08713692426681519, 0.07091736048460007] \n",
      "epoch[20] iter[80] loss: [0.11246788501739502, 0.10097718238830566] \n",
      "epoch[20] iter[120] loss: [0.11130326986312866, 0.10355272144079208] \n",
      "epoch[20] iter[160] loss: [0.09085942804813385, 0.0846058577299118] \n",
      "epoch[20] iter[200] loss: [0.09249616414308548, 0.09950986504554749] \n",
      "epoch[20] iter[240] loss: [0.09444768726825714, 0.06945881992578506] \n",
      "epoch[20] iter[280] loss: [0.08529621362686157, 0.1016191616654396] \n",
      "epoch[20] iter[320] loss: [0.09273693710565567, 0.07210848480463028] \n",
      "epoch[20] iter[360] loss: [0.08328169584274292, 0.07004930824041367] \n",
      "epoch[20] iter[400] loss: [0.09325301647186279, 0.07167540490627289] \n",
      "epoch[20] iter[440] loss: [0.08674517273902893, 0.06918066740036011] \n",
      "epoch[20] iter[480] loss: [0.07834581285715103, 0.06323938816785812] \n",
      "epoch[20] iter[520] loss: [0.09626580774784088, 0.10017421096563339] \n",
      "epoch[20] iter[560] loss: [0.09374195337295532, 0.09489733725786209] \n",
      "epoch[20] iter[600] loss: [0.10953399538993835, 0.0931074395775795] \n",
      "epoch[20] iter[640] loss: [0.09615284204483032, 0.09638728946447372] \n",
      "epoch[20] iter[680] loss: [0.08664683997631073, 0.08208337426185608] \n",
      "epoch[20] iter[720] loss: [0.09374698251485825, 0.0725734531879425] \n",
      "epoch[20] iter[760] loss: [0.09306526929140091, 0.06647803634405136] \n",
      "Finish epoch[20], time elapsed[571.3752512931824]\n",
      "epoch[21] iter[0] loss: [0.0821940004825592, 0.06485404819250107] \n",
      "epoch[21] iter[40] loss: [0.09226638823747635, 0.11755876243114471] \n",
      "epoch[21] iter[80] loss: [0.09770804643630981, 0.0765763521194458] \n",
      "epoch[21] iter[120] loss: [0.08090043067932129, 0.08034735172986984] \n",
      "epoch[21] iter[160] loss: [0.08257260918617249, 0.07633855193853378] \n",
      "epoch[21] iter[200] loss: [0.10346619039773941, 0.08486192673444748] \n",
      "epoch[21] iter[240] loss: [0.08715016394853592, 0.0647161453962326] \n",
      "epoch[21] iter[280] loss: [0.08119580894708633, 0.06341028958559036] \n",
      "epoch[21] iter[320] loss: [0.08895698189735413, 0.08300548791885376] \n",
      "epoch[21] iter[360] loss: [0.081450916826725, 0.07027089595794678] \n",
      "epoch[21] iter[400] loss: [0.09473893791437149, 0.08171052485704422] \n",
      "epoch[21] iter[440] loss: [0.09818889945745468, 0.06662030518054962] \n",
      "epoch[21] iter[480] loss: [0.0913853868842125, 0.06421776115894318] \n",
      "epoch[21] iter[520] loss: [0.07612936198711395, 0.055960141122341156] \n",
      "epoch[21] iter[560] loss: [0.09215997904539108, 0.07514020800590515] \n",
      "epoch[21] iter[600] loss: [0.08531150221824646, 0.06594477593898773] \n",
      "epoch[21] iter[640] loss: [0.07955359667539597, 0.07109305262565613] \n",
      "epoch[21] iter[680] loss: [0.08548976480960846, 0.07667110115289688] \n",
      "epoch[21] iter[720] loss: [0.09577120840549469, 0.08016686141490936] \n",
      "epoch[21] iter[760] loss: [0.10080783814191818, 0.10683809965848923] \n",
      "Finish epoch[21], time elapsed[572.6107604503632]\n",
      "epoch[22] iter[0] loss: [0.07954994589090347, 0.12122584134340286] \n",
      "epoch[22] iter[40] loss: [0.102402463555336, 0.07861801236867905] \n",
      "epoch[22] iter[80] loss: [0.08574255555868149, 0.06509614735841751] \n",
      "epoch[22] iter[120] loss: [0.07877632975578308, 0.06610140204429626] \n",
      "epoch[22] iter[160] loss: [0.09056443721055984, 0.06370583176612854] \n",
      "epoch[22] iter[200] loss: [0.08308006823062897, 0.08249597996473312] \n",
      "epoch[22] iter[240] loss: [0.0809812918305397, 0.059866782277822495] \n",
      "epoch[22] iter[280] loss: [0.0850900411605835, 0.06603392213582993] \n",
      "epoch[22] iter[320] loss: [0.08607350289821625, 0.0678924098610878] \n",
      "epoch[22] iter[360] loss: [0.09875964373350143, 0.08278898894786835] \n",
      "epoch[22] iter[400] loss: [0.0781548023223877, 0.06567969918251038] \n",
      "epoch[22] iter[440] loss: [0.09896475821733475, 0.07089873403310776] \n",
      "epoch[22] iter[480] loss: [0.08905524760484695, 0.07565627247095108] \n",
      "epoch[22] iter[520] loss: [0.10297966748476028, 0.08572517335414886] \n",
      "epoch[22] iter[560] loss: [0.0862082913517952, 0.07043273746967316] \n",
      "epoch[22] iter[600] loss: [0.08274569362401962, 0.06897672265768051] \n",
      "epoch[22] iter[640] loss: [0.08686073124408722, 0.06857889145612717] \n",
      "epoch[22] iter[680] loss: [0.0868186503648758, 0.07920908182859421] \n",
      "epoch[22] iter[720] loss: [0.08024745434522629, 0.06552635878324509] \n",
      "epoch[22] iter[760] loss: [0.07846558839082718, 0.0632430762052536] \n",
      "Finish epoch[22], time elapsed[570.7112834453583]\n",
      "epoch[23] iter[0] loss: [0.08031775802373886, 0.06742219626903534] \n",
      "epoch[23] iter[40] loss: [0.08506236225366592, 0.12496301531791687] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch[23] iter[80] loss: [0.08199658989906311, 0.07215652614831924] \n",
      "epoch[23] iter[120] loss: [0.08869930356740952, 0.06802044808864594] \n",
      "epoch[23] iter[160] loss: [0.06934434175491333, 0.06117093563079834] \n",
      "epoch[23] iter[200] loss: [0.07950741797685623, 0.06140842288732529] \n",
      "epoch[23] iter[240] loss: [0.08917465060949326, 0.08565214276313782] \n",
      "epoch[23] iter[280] loss: [0.08249933272600174, 0.09138937294483185] \n",
      "epoch[23] iter[320] loss: [0.08285845071077347, 0.06629472970962524] \n",
      "epoch[23] iter[360] loss: [0.09136195480823517, 0.07817798107862473] \n",
      "epoch[23] iter[400] loss: [0.08992115408182144, 0.06887319684028625] \n",
      "epoch[23] iter[440] loss: [0.08815968036651611, 0.10622882843017578] \n",
      "epoch[23] iter[480] loss: [0.09348545223474503, 0.06612003594636917] \n",
      "epoch[23] iter[520] loss: [0.08970817178487778, 0.0704876109957695] \n",
      "epoch[23] iter[560] loss: [0.0857674777507782, 0.07462014257907867] \n",
      "epoch[23] iter[600] loss: [0.07903894037008286, 0.08495820313692093] \n",
      "epoch[23] iter[640] loss: [0.09436593949794769, 0.08118323236703873] \n",
      "epoch[23] iter[680] loss: [0.08180123567581177, 0.0623072013258934] \n",
      "epoch[23] iter[720] loss: [0.07780565321445465, 0.060334887355566025] \n",
      "epoch[23] iter[760] loss: [0.10550110042095184, 0.08037891238927841] \n",
      "Finish epoch[23], time elapsed[570.8519752025604]\n",
      "epoch[24] iter[0] loss: [0.0813419297337532, 0.0595594123005867] \n",
      "epoch[24] iter[40] loss: [0.08541931957006454, 0.06917538493871689] \n",
      "epoch[24] iter[80] loss: [0.09658335894346237, 0.07255738973617554] \n",
      "epoch[24] iter[120] loss: [0.09525295346975327, 0.09545378386974335] \n",
      "epoch[24] iter[160] loss: [0.08219700306653976, 0.06557181477546692] \n",
      "epoch[24] iter[200] loss: [0.07348766922950745, 0.06213139742612839] \n",
      "epoch[24] iter[240] loss: [0.08374982327222824, 0.07564418762922287] \n",
      "epoch[24] iter[280] loss: [0.07833520323038101, 0.06405700743198395] \n",
      "epoch[24] iter[320] loss: [0.07485426217317581, 0.05763597786426544] \n",
      "epoch[24] iter[360] loss: [0.08710812032222748, 0.06190355867147446] \n",
      "epoch[24] iter[400] loss: [0.08352886140346527, 0.08302731066942215] \n",
      "epoch[24] iter[440] loss: [0.07385827600955963, 0.05553847551345825] \n",
      "epoch[24] iter[480] loss: [0.07152514159679413, 0.05614838749170303] \n",
      "epoch[24] iter[520] loss: [0.10486842691898346, 0.10327709466218948] \n",
      "epoch[24] iter[560] loss: [0.07674048840999603, 0.061384934931993484] \n",
      "epoch[24] iter[600] loss: [0.07756446301937103, 0.06221296265721321] \n",
      "epoch[24] iter[640] loss: [0.09024802595376968, 0.06187872961163521] \n",
      "epoch[24] iter[680] loss: [0.07748483866453171, 0.06168537959456444] \n",
      "epoch[24] iter[720] loss: [0.07492265105247498, 0.05572091042995453] \n",
      "epoch[24] iter[760] loss: [0.10249090194702148, 0.0821123793721199] \n",
      "Finish epoch[24], time elapsed[571.00066447258]\n",
      "epoch[25] iter[0] loss: [0.07375892996788025, 0.06681590527296066] \n",
      "epoch[25] iter[40] loss: [0.08929046988487244, 0.08350425958633423] \n",
      "epoch[25] iter[80] loss: [0.0984208956360817, 0.06932945549488068] \n",
      "epoch[25] iter[120] loss: [0.10070480406284332, 0.07939358800649643] \n",
      "epoch[25] iter[160] loss: [0.09494423866271973, 0.0756247416138649] \n",
      "epoch[25] iter[200] loss: [0.08256018161773682, 0.08246766030788422] \n",
      "epoch[25] iter[240] loss: [0.09255031496286392, 0.072104312479496] \n",
      "epoch[25] iter[280] loss: [0.08895406126976013, 0.06816964596509933] \n",
      "epoch[25] iter[320] loss: [0.08317678421735764, 0.06440923362970352] \n",
      "epoch[25] iter[360] loss: [0.08168347924947739, 0.06375735998153687] \n",
      "epoch[25] iter[400] loss: [0.0783160924911499, 0.05904960259795189] \n",
      "epoch[25] iter[440] loss: [0.09141498059034348, 0.08211463689804077] \n",
      "epoch[25] iter[480] loss: [0.07762293517589569, 0.06382521986961365] \n",
      "epoch[25] iter[520] loss: [0.08159155398607254, 0.06052197143435478] \n",
      "epoch[25] iter[560] loss: [0.08606040477752686, 0.06326155364513397] \n",
      "epoch[25] iter[600] loss: [0.09288739413022995, 0.07209226489067078] \n",
      "epoch[25] iter[640] loss: [0.08790300041437149, 0.0645255371928215] \n",
      "epoch[25] iter[680] loss: [0.10447792708873749, 0.09177947044372559] \n",
      "epoch[25] iter[720] loss: [0.09009455144405365, 0.070571668446064] \n",
      "epoch[25] iter[760] loss: [0.07435078918933868, 0.055237777531147] \n",
      "Finish epoch[25], time elapsed[569.9701228141785]\n",
      "epoch[26] iter[0] loss: [0.08630342781543732, 0.07373520731925964] \n",
      "epoch[26] iter[40] loss: [0.08869840949773788, 0.05622332543134689] \n",
      "epoch[26] iter[80] loss: [0.08249998837709427, 0.07844549417495728] \n",
      "epoch[26] iter[120] loss: [0.08463075011968613, 0.059382114559412] \n",
      "epoch[26] iter[160] loss: [0.08415986597537994, 0.07025956362485886] \n",
      "epoch[26] iter[200] loss: [0.07191535085439682, 0.05679239705204964] \n",
      "epoch[26] iter[240] loss: [0.08353643864393234, 0.08673715591430664] \n",
      "epoch[26] iter[280] loss: [0.07835855334997177, 0.05568625032901764] \n",
      "epoch[26] iter[320] loss: [0.09311459213495255, 0.08140479028224945] \n",
      "epoch[26] iter[360] loss: [0.08556773513555527, 0.06290914118289948] \n",
      "epoch[26] iter[400] loss: [0.0820942223072052, 0.06491298228502274] \n",
      "epoch[26] iter[440] loss: [0.09222346544265747, 0.07681362330913544] \n",
      "epoch[26] iter[480] loss: [0.09250890463590622, 0.06921722739934921] \n",
      "epoch[26] iter[520] loss: [0.09100299328565598, 0.06709521263837814] \n",
      "epoch[26] iter[560] loss: [0.11408042907714844, 0.10322859138250351] \n",
      "epoch[26] iter[600] loss: [0.07755725830793381, 0.06007891148328781] \n",
      "epoch[26] iter[640] loss: [0.07857287675142288, 0.06259602308273315] \n",
      "epoch[26] iter[680] loss: [0.07992678135633469, 0.0857718288898468] \n",
      "epoch[26] iter[720] loss: [0.07769427448511124, 0.056737020611763] \n",
      "epoch[26] iter[760] loss: [0.0808153748512268, 0.06707359850406647] \n",
      "Finish epoch[26], time elapsed[566.7237389087677]\n",
      "epoch[27] iter[0] loss: [0.08099286258220673, 0.06595070660114288] \n",
      "epoch[27] iter[40] loss: [0.08004925400018692, 0.059929415583610535] \n",
      "epoch[27] iter[80] loss: [0.10709177702665329, 0.08731137961149216] \n",
      "epoch[27] iter[120] loss: [0.08118239790201187, 0.05897288769483566] \n",
      "epoch[27] iter[160] loss: [0.08864133805036545, 0.07577848434448242] \n",
      "epoch[27] iter[200] loss: [0.07324133813381195, 0.05532347783446312] \n",
      "epoch[27] iter[240] loss: [0.08508624136447906, 0.06893978267908096] \n",
      "epoch[27] iter[280] loss: [0.07737241685390472, 0.06830790638923645] \n",
      "epoch[27] iter[320] loss: [0.08062507957220078, 0.05707445368170738] \n",
      "epoch[27] iter[360] loss: [0.07128589600324631, 0.08437620103359222] \n",
      "epoch[27] iter[400] loss: [0.07815395295619965, 0.06751135736703873] \n",
      "epoch[27] iter[440] loss: [0.09081629663705826, 0.06898903101682663] \n",
      "epoch[27] iter[480] loss: [0.08343804627656937, 0.06045433133840561] \n",
      "epoch[27] iter[520] loss: [0.08631862699985504, 0.05925419181585312] \n",
      "epoch[27] iter[560] loss: [0.10848421603441238, 0.09115532040596008] \n",
      "epoch[27] iter[600] loss: [0.08486373722553253, 0.0747198686003685] \n",
      "epoch[27] iter[640] loss: [0.08484558761119843, 0.06380034983158112] \n",
      "epoch[27] iter[680] loss: [0.0773657038807869, 0.05785120651125908] \n",
      "epoch[27] iter[720] loss: [0.07766394317150116, 0.06372035294771194] \n",
      "epoch[27] iter[760] loss: [0.07727978378534317, 0.06781374663114548] \n",
      "Finish epoch[27], time elapsed[565.331869840622]\n",
      "epoch[28] iter[0] loss: [0.08300578594207764, 0.06441601365804672] \n",
      "epoch[28] iter[40] loss: [0.08790221810340881, 0.061641305685043335] \n",
      "epoch[28] iter[80] loss: [0.08947327733039856, 0.061405330896377563] \n",
      "epoch[28] iter[120] loss: [0.08654596656560898, 0.06961052119731903] \n",
      "epoch[28] iter[160] loss: [0.07497311383485794, 0.05284033343195915] \n",
      "epoch[28] iter[200] loss: [0.08502794057130814, 0.06544964015483856] \n",
      "epoch[28] iter[240] loss: [0.08324946463108063, 0.06250082701444626] \n",
      "epoch[28] iter[280] loss: [0.07586219906806946, 0.05929246172308922] \n",
      "epoch[28] iter[320] loss: [0.07805249840021133, 0.058265578001737595] \n",
      "epoch[28] iter[360] loss: [0.06936736404895782, 0.06112835928797722] \n",
      "epoch[28] iter[400] loss: [0.07533787190914154, 0.063533253967762] \n",
      "epoch[28] iter[440] loss: [0.0877528190612793, 0.07031808793544769] \n",
      "epoch[28] iter[480] loss: [0.0822385922074318, 0.06511105597019196] \n",
      "epoch[28] iter[520] loss: [0.07803495973348618, 0.05666358768939972] \n",
      "epoch[28] iter[560] loss: [0.08066251873970032, 0.06292630732059479] \n",
      "epoch[28] iter[600] loss: [0.08110407739877701, 0.06180722638964653] \n",
      "epoch[28] iter[640] loss: [0.07858995348215103, 0.05671769380569458] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch[28] iter[680] loss: [0.08172861486673355, 0.05636866018176079] \n",
      "epoch[28] iter[720] loss: [0.07589983940124512, 0.051847703754901886] \n",
      "epoch[28] iter[760] loss: [0.10252660512924194, 0.09366823732852936] \n",
      "Finish epoch[28], time elapsed[562.1930923461914]\n",
      "epoch[29] iter[0] loss: [0.08004079759120941, 0.07112443447113037] \n",
      "epoch[29] iter[40] loss: [0.08229754865169525, 0.060033537447452545] \n",
      "epoch[29] iter[80] loss: [0.06496092677116394, 0.04883505776524544] \n",
      "epoch[29] iter[120] loss: [0.07688969373703003, 0.058737825602293015] \n",
      "epoch[29] iter[160] loss: [0.07649166136980057, 0.05614001303911209] \n",
      "epoch[29] iter[200] loss: [0.08918707817792892, 0.0792703926563263] \n",
      "epoch[29] iter[240] loss: [0.08698274195194244, 0.07213940471410751] \n",
      "epoch[29] iter[280] loss: [0.08453754335641861, 0.05927702784538269] \n",
      "epoch[29] iter[320] loss: [0.0796503946185112, 0.05838026478886604] \n",
      "epoch[29] iter[360] loss: [0.08185742795467377, 0.09166701883077621] \n",
      "epoch[29] iter[400] loss: [0.07931020110845566, 0.07428912073373795] \n",
      "epoch[29] iter[440] loss: [0.08270443975925446, 0.07151549309492111] \n",
      "epoch[29] iter[480] loss: [0.08192328363656998, 0.06735292077064514] \n",
      "epoch[29] iter[520] loss: [0.08388280868530273, 0.05926790460944176] \n",
      "epoch[29] iter[560] loss: [0.08415816724300385, 0.0743342936038971] \n",
      "epoch[29] iter[600] loss: [0.08472955971956253, 0.058860957622528076] \n",
      "epoch[29] iter[640] loss: [0.08252590149641037, 0.06719297170639038] \n",
      "epoch[29] iter[680] loss: [0.09547897428274155, 0.06706973910331726] \n",
      "epoch[29] iter[720] loss: [0.09484922885894775, 0.06659569591283798] \n",
      "epoch[29] iter[760] loss: [0.0805017352104187, 0.054320257157087326] \n",
      "Finish epoch[29], time elapsed[559.8680438995361]\n",
      "epoch[30] iter[0] loss: [0.09919819235801697, 0.08493528515100479] \n",
      "epoch[30] iter[40] loss: [0.08945067226886749, 0.0847877785563469] \n",
      "epoch[30] iter[80] loss: [0.09455173462629318, 0.07165002822875977] \n",
      "epoch[30] iter[120] loss: [0.07757008820772171, 0.06421999633312225] \n",
      "epoch[30] iter[160] loss: [0.07529273629188538, 0.0643085241317749] \n",
      "epoch[30] iter[200] loss: [0.07715582847595215, 0.06684209406375885] \n",
      "epoch[30] iter[240] loss: [0.07928024232387543, 0.05896777659654617] \n",
      "epoch[30] iter[280] loss: [0.089248426258564, 0.07051726430654526] \n",
      "epoch[30] iter[320] loss: [0.0957413911819458, 0.06930648535490036] \n",
      "epoch[30] iter[360] loss: [0.08588007092475891, 0.06767572462558746] \n",
      "epoch[30] iter[400] loss: [0.08291839063167572, 0.07172314822673798] \n",
      "epoch[30] iter[440] loss: [0.07678546011447906, 0.05455427244305611] \n",
      "epoch[30] iter[480] loss: [0.08700273185968399, 0.06297438591718674] \n",
      "epoch[30] iter[520] loss: [0.06850583851337433, 0.052630625665187836] \n",
      "epoch[30] iter[560] loss: [0.08250746876001358, 0.07534844428300858] \n",
      "epoch[30] iter[600] loss: [0.0831562727689743, 0.07203369587659836] \n",
      "epoch[30] iter[640] loss: [0.07349725067615509, 0.058840472251176834] \n",
      "epoch[30] iter[680] loss: [0.08255837857723236, 0.0732472836971283] \n",
      "epoch[30] iter[720] loss: [0.0687142163515091, 0.05166110023856163] \n",
      "epoch[30] iter[760] loss: [0.08717420697212219, 0.0728912279009819] \n",
      "Finish epoch[30], time elapsed[558.4312934875488]\n",
      "epoch[31] iter[0] loss: [0.08334407210350037, 0.06834748387336731] \n",
      "epoch[31] iter[40] loss: [0.08025580644607544, 0.06467258930206299] \n",
      "epoch[31] iter[80] loss: [0.0864109992980957, 0.06421259045600891] \n",
      "epoch[31] iter[120] loss: [0.095511794090271, 0.07784802466630936] \n",
      "epoch[31] iter[160] loss: [0.0754346027970314, 0.05440933629870415] \n",
      "epoch[31] iter[200] loss: [0.07543070614337921, 0.0618990883231163] \n",
      "epoch[31] iter[240] loss: [0.07342761754989624, 0.06222301349043846] \n",
      "epoch[31] iter[280] loss: [0.07405944168567657, 0.06436533480882645] \n",
      "epoch[31] iter[320] loss: [0.08682521432638168, 0.07017776370048523] \n",
      "epoch[31] iter[360] loss: [0.07119366526603699, 0.054130420088768005] \n",
      "epoch[31] iter[400] loss: [0.07644802331924438, 0.0616137757897377] \n",
      "epoch[31] iter[440] loss: [0.06621462106704712, 0.05151001736521721] \n",
      "epoch[31] iter[480] loss: [0.07728955149650574, 0.07015123218297958] \n",
      "epoch[31] iter[520] loss: [0.08413255214691162, 0.06746354699134827] \n",
      "epoch[31] iter[560] loss: [0.0839095413684845, 0.07110865414142609] \n",
      "epoch[31] iter[600] loss: [0.06835958361625671, 0.05288510024547577] \n",
      "epoch[31] iter[640] loss: [0.07088369131088257, 0.051678478717803955] \n",
      "epoch[31] iter[680] loss: [0.08451738208532333, 0.06550491601228714] \n",
      "epoch[31] iter[720] loss: [0.06902950257062912, 0.05070861056447029] \n",
      "epoch[31] iter[760] loss: [0.09509249776601791, 0.06781145930290222] \n",
      "Finish epoch[31], time elapsed[560.6629571914673]\n",
      "epoch[32] iter[0] loss: [0.07550366222858429, 0.054276347160339355] \n",
      "epoch[32] iter[40] loss: [0.08847120404243469, 0.05903796851634979] \n",
      "epoch[32] iter[80] loss: [0.08126357942819595, 0.07260105758905411] \n",
      "epoch[32] iter[120] loss: [0.07454361766576767, 0.05345052108168602] \n",
      "epoch[32] iter[160] loss: [0.07587537914514542, 0.06619025021791458] \n",
      "epoch[32] iter[200] loss: [0.08063824474811554, 0.05799843743443489] \n",
      "epoch[32] iter[240] loss: [0.07579126954078674, 0.058243490755558014] \n",
      "epoch[32] iter[280] loss: [0.0855369046330452, 0.056840211153030396] \n",
      "epoch[32] iter[320] loss: [0.08184026926755905, 0.059565916657447815] \n",
      "epoch[32] iter[360] loss: [0.07696357369422913, 0.060083553194999695] \n",
      "epoch[32] iter[400] loss: [0.08037862181663513, 0.05771692469716072] \n",
      "epoch[32] iter[440] loss: [0.08098482340574265, 0.06025046482682228] \n",
      "epoch[32] iter[480] loss: [0.09473040699958801, 0.06831830739974976] \n",
      "epoch[32] iter[520] loss: [0.07210516929626465, 0.06337261945009232] \n",
      "epoch[32] iter[560] loss: [0.07595659792423248, 0.06086067855358124] \n",
      "epoch[32] iter[600] loss: [0.08780204504728317, 0.05783408135175705] \n",
      "epoch[32] iter[640] loss: [0.08937819302082062, 0.06967058032751083] \n",
      "epoch[32] iter[680] loss: [0.06872092187404633, 0.05673237517476082] \n",
      "epoch[32] iter[720] loss: [0.08041328191757202, 0.06310977041721344] \n",
      "epoch[32] iter[760] loss: [0.08595045655965805, 0.06606277078390121] \n",
      "Finish epoch[32], time elapsed[560.8553516864777]\n",
      "epoch[33] iter[0] loss: [0.07020501792430878, 0.05998669192194939] \n",
      "epoch[33] iter[40] loss: [0.08667747676372528, 0.06299090385437012] \n",
      "epoch[33] iter[80] loss: [0.09055351465940475, 0.06697551161050797] \n",
      "epoch[33] iter[120] loss: [0.07921753078699112, 0.0577104315161705] \n",
      "epoch[33] iter[160] loss: [0.08829059451818466, 0.08772795647382736] \n",
      "epoch[33] iter[200] loss: [0.08785300701856613, 0.060638077557086945] \n",
      "epoch[33] iter[240] loss: [0.07808542251586914, 0.05451586842536926] \n",
      "epoch[33] iter[280] loss: [0.08912120014429092, 0.057880379259586334] \n",
      "epoch[33] iter[320] loss: [0.07319021970033646, 0.058951638638973236] \n",
      "epoch[33] iter[360] loss: [0.0700531080365181, 0.052027828991413116] \n",
      "epoch[33] iter[400] loss: [0.06790204346179962, 0.06281381845474243] \n",
      "epoch[33] iter[440] loss: [0.0863095223903656, 0.0539710596203804] \n",
      "epoch[33] iter[480] loss: [0.07573875039815903, 0.057961732149124146] \n",
      "epoch[33] iter[520] loss: [0.08587253093719482, 0.06397020071744919] \n",
      "epoch[33] iter[560] loss: [0.07309254258871078, 0.052102088928222656] \n",
      "epoch[33] iter[600] loss: [0.08680719137191772, 0.06238904595375061] \n",
      "epoch[33] iter[640] loss: [0.0683431401848793, 0.05855698511004448] \n",
      "epoch[33] iter[680] loss: [0.07884837687015533, 0.06612332910299301] \n",
      "epoch[33] iter[720] loss: [0.07892464846372604, 0.060354460030794144] \n",
      "epoch[33] iter[760] loss: [0.07541941106319427, 0.056792572140693665] \n",
      "Finish epoch[33], time elapsed[559.1539115905762]\n",
      "epoch[34] iter[0] loss: [0.07045280933380127, 0.058488842099905014] \n",
      "epoch[34] iter[40] loss: [0.08223296701908112, 0.06408119201660156] \n",
      "epoch[34] iter[80] loss: [0.09212568402290344, 0.06561821699142456] \n",
      "epoch[34] iter[120] loss: [0.07603224366903305, 0.06638370454311371] \n",
      "epoch[34] iter[160] loss: [0.0731336921453476, 0.0852719247341156] \n",
      "epoch[34] iter[200] loss: [0.07274887710809708, 0.055899620056152344] \n",
      "epoch[34] iter[240] loss: [0.08192864805459976, 0.08857899904251099] \n",
      "epoch[34] iter[280] loss: [0.07879103720188141, 0.05675018951296806] \n",
      "epoch[34] iter[320] loss: [0.0895794928073883, 0.07470591366291046] \n",
      "epoch[34] iter[360] loss: [0.07101118564605713, 0.05011691153049469] \n",
      "epoch[34] iter[400] loss: [0.08704283088445663, 0.06915564090013504] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch[34] iter[440] loss: [0.08593376725912094, 0.08203341066837311] \n",
      "epoch[34] iter[480] loss: [0.07624319940805435, 0.07029964029788971] \n",
      "epoch[34] iter[520] loss: [0.07870279997587204, 0.0598335862159729] \n",
      "epoch[34] iter[560] loss: [0.08336634188890457, 0.064451664686203] \n",
      "epoch[34] iter[600] loss: [0.07398582994937897, 0.05706215649843216] \n",
      "epoch[34] iter[640] loss: [0.08660975843667984, 0.06680729240179062] \n",
      "epoch[34] iter[680] loss: [0.07226915657520294, 0.07883460819721222] \n",
      "epoch[34] iter[720] loss: [0.08149158954620361, 0.05658898875117302] \n",
      "epoch[34] iter[760] loss: [0.08312855660915375, 0.0703616663813591] \n",
      "Finish epoch[34], time elapsed[560.6405203342438]\n",
      "epoch[35] iter[0] loss: [0.0795765146613121, 0.05994245037436485] \n",
      "epoch[35] iter[40] loss: [0.07942330092191696, 0.06219873204827309] \n",
      "epoch[35] iter[80] loss: [0.06834916025400162, 0.05724059045314789] \n",
      "epoch[35] iter[120] loss: [0.09078642725944519, 0.06166950985789299] \n",
      "epoch[35] iter[160] loss: [0.07915206253528595, 0.05795442685484886] \n",
      "epoch[35] iter[200] loss: [0.0685618594288826, 0.05397553741931915] \n",
      "epoch[35] iter[240] loss: [0.07912184298038483, 0.06449101865291595] \n",
      "epoch[35] iter[280] loss: [0.06858597695827484, 0.07479020953178406] \n",
      "epoch[35] iter[320] loss: [0.07596150785684586, 0.0681581124663353] \n",
      "epoch[35] iter[360] loss: [0.08206827193498611, 0.09039584547281265] \n",
      "epoch[35] iter[400] loss: [0.07256806641817093, 0.04911351948976517] \n",
      "epoch[35] iter[440] loss: [0.09239324182271957, 0.09334354847669601] \n",
      "epoch[35] iter[480] loss: [0.07157161831855774, 0.052997663617134094] \n",
      "epoch[35] iter[520] loss: [0.07837940752506256, 0.06116650626063347] \n",
      "epoch[35] iter[560] loss: [0.0782921314239502, 0.05502263084053993] \n",
      "epoch[35] iter[600] loss: [0.06636511534452438, 0.04992404952645302] \n",
      "epoch[35] iter[640] loss: [0.08367423713207245, 0.05776670575141907] \n",
      "epoch[35] iter[680] loss: [0.08030415326356888, 0.06783922016620636] \n",
      "epoch[35] iter[720] loss: [0.0685291513800621, 0.05564640834927559] \n",
      "epoch[35] iter[760] loss: [0.07168623059988022, 0.05238056182861328] \n",
      "Finish epoch[35], time elapsed[558.9999377727509]\n",
      "epoch[36] iter[0] loss: [0.07387031614780426, 0.05654266104102135] \n",
      "epoch[36] iter[40] loss: [0.06880533695220947, 0.05816545709967613] \n",
      "epoch[36] iter[80] loss: [0.07304441928863525, 0.05572224780917168] \n",
      "epoch[36] iter[120] loss: [0.08134724944829941, 0.057209182530641556] \n",
      "epoch[36] iter[160] loss: [0.07572559267282486, 0.05713701620697975] \n",
      "epoch[36] iter[200] loss: [0.07953895628452301, 0.055911820381879807] \n",
      "epoch[36] iter[240] loss: [0.0710921436548233, 0.05735437944531441] \n",
      "epoch[36] iter[280] loss: [0.07526031881570816, 0.06865908950567245] \n",
      "epoch[36] iter[320] loss: [0.0748598501086235, 0.0575118325650692] \n",
      "epoch[36] iter[360] loss: [0.07560412585735321, 0.05921584740281105] \n",
      "epoch[36] iter[400] loss: [0.07706817984580994, 0.06253715604543686] \n",
      "epoch[36] iter[440] loss: [0.0709867849946022, 0.04987134784460068] \n",
      "epoch[36] iter[480] loss: [0.0819682776927948, 0.07517247647047043] \n",
      "epoch[36] iter[520] loss: [0.08140113949775696, 0.061731789261102676] \n",
      "epoch[36] iter[560] loss: [0.07399527728557587, 0.05386490747332573] \n",
      "epoch[36] iter[600] loss: [0.07139614224433899, 0.056789323687553406] \n",
      "epoch[36] iter[640] loss: [0.07397135347127914, 0.06474805623292923] \n",
      "epoch[36] iter[680] loss: [0.08507630974054337, 0.06285102665424347] \n",
      "epoch[36] iter[720] loss: [0.08587919175624847, 0.0652116909623146] \n",
      "epoch[36] iter[760] loss: [0.08505430072546005, 0.06801886856555939] \n",
      "Finish epoch[36], time elapsed[568.8445580005646]\n",
      "epoch[37] iter[0] loss: [0.07519253343343735, 0.07823197543621063] \n",
      "epoch[37] iter[40] loss: [0.08029011636972427, 0.06060970202088356] \n",
      "epoch[37] iter[80] loss: [0.07101380079984665, 0.05980273708701134] \n",
      "epoch[37] iter[120] loss: [0.06546229124069214, 0.048759154975414276] \n",
      "epoch[37] iter[160] loss: [0.07959451526403427, 0.07420914620161057] \n",
      "epoch[37] iter[200] loss: [0.07215650379657745, 0.054321739822626114] \n",
      "epoch[37] iter[240] loss: [0.07865753769874573, 0.06133206561207771] \n",
      "epoch[37] iter[280] loss: [0.08018770813941956, 0.0829266607761383] \n",
      "epoch[37] iter[320] loss: [0.06509368121623993, 0.04694311320781708] \n",
      "epoch[37] iter[360] loss: [0.08134686946868896, 0.07902514934539795] \n",
      "epoch[37] iter[400] loss: [0.0866929143667221, 0.06450565159320831] \n",
      "epoch[37] iter[440] loss: [0.07789187878370285, 0.06261490285396576] \n",
      "epoch[37] iter[480] loss: [0.06328483670949936, 0.04887112230062485] \n",
      "epoch[37] iter[520] loss: [0.08618702739477158, 0.07401248812675476] \n",
      "epoch[37] iter[560] loss: [0.07882754504680634, 0.06881697475910187] \n",
      "epoch[37] iter[600] loss: [0.07917233556509018, 0.05414868891239166] \n",
      "epoch[37] iter[640] loss: [0.07352451235055923, 0.049222271889448166] \n",
      "epoch[37] iter[680] loss: [0.07285309582948685, 0.053731389343738556] \n",
      "epoch[37] iter[720] loss: [0.0833205133676529, 0.05652490630745888] \n",
      "epoch[37] iter[760] loss: [0.07122714817523956, 0.05350398272275925] \n",
      "Finish epoch[37], time elapsed[570.2500197887421]\n",
      "epoch[38] iter[0] loss: [0.07618360221385956, 0.06777047365903854] \n",
      "epoch[38] iter[40] loss: [0.06779695302248001, 0.048441264778375626] \n",
      "epoch[38] iter[80] loss: [0.08028526604175568, 0.05876749008893967] \n",
      "epoch[38] iter[120] loss: [0.07462824881076813, 0.05758902430534363] \n",
      "epoch[38] iter[160] loss: [0.06866158545017242, 0.052686311304569244] \n",
      "epoch[38] iter[200] loss: [0.06992077082395554, 0.05276905745267868] \n",
      "epoch[38] iter[240] loss: [0.07309067249298096, 0.0591442734003067] \n",
      "epoch[38] iter[280] loss: [0.07595761120319366, 0.05386393144726753] \n",
      "epoch[38] iter[320] loss: [0.07772724330425262, 0.06552854925394058] \n",
      "epoch[38] iter[360] loss: [0.0729394406080246, 0.05893876031041145] \n",
      "epoch[38] iter[400] loss: [0.08272244781255722, 0.05932117626070976] \n",
      "epoch[38] iter[440] loss: [0.07364043593406677, 0.06516627967357635] \n",
      "epoch[38] iter[480] loss: [0.06919664144515991, 0.055777549743652344] \n",
      "epoch[38] iter[520] loss: [0.07895881682634354, 0.05391727387905121] \n",
      "epoch[38] iter[560] loss: [0.07899409532546997, 0.057841796427965164] \n",
      "epoch[38] iter[600] loss: [0.06837978959083557, 0.050768788903951645] \n",
      "epoch[38] iter[640] loss: [0.07245206087827682, 0.05315317213535309] \n",
      "epoch[38] iter[680] loss: [0.08052364736795425, 0.057230617851018906] \n",
      "epoch[38] iter[720] loss: [0.08486364781856537, 0.06850109249353409] \n",
      "epoch[38] iter[760] loss: [0.06384939700365067, 0.04523741453886032] \n",
      "Finish epoch[38], time elapsed[572.3072571754456]\n",
      "epoch[39] iter[0] loss: [0.07135967165231705, 0.051060423254966736] \n",
      "epoch[39] iter[40] loss: [0.08431047946214676, 0.056909091770648956] \n",
      "epoch[39] iter[80] loss: [0.07366762310266495, 0.055622413754463196] \n",
      "epoch[39] iter[120] loss: [0.08154992014169693, 0.05542103946208954] \n",
      "epoch[39] iter[160] loss: [0.08699944615364075, 0.062219683080911636] \n",
      "epoch[39] iter[200] loss: [0.07586520165205002, 0.04973156750202179] \n",
      "epoch[39] iter[240] loss: [0.08997531235218048, 0.05840959772467613] \n",
      "epoch[39] iter[280] loss: [0.08004710078239441, 0.06689220666885376] \n",
      "epoch[39] iter[320] loss: [0.0711134672164917, 0.05278686434030533] \n",
      "epoch[39] iter[360] loss: [0.07546379417181015, 0.05292384698987007] \n",
      "epoch[39] iter[400] loss: [0.07710746675729752, 0.04967914894223213] \n",
      "epoch[39] iter[440] loss: [0.0706334337592125, 0.05336938425898552] \n",
      "epoch[39] iter[480] loss: [0.08449792861938477, 0.060876235365867615] \n",
      "epoch[39] iter[520] loss: [0.07478468865156174, 0.059053968638181686] \n",
      "epoch[39] iter[560] loss: [0.08628915250301361, 0.07083183526992798] \n",
      "epoch[39] iter[600] loss: [0.07567679136991501, 0.052478447556495667] \n",
      "epoch[39] iter[640] loss: [0.07427644729614258, 0.06583360582590103] \n",
      "epoch[39] iter[680] loss: [0.077070452272892, 0.07079454511404037] \n",
      "epoch[39] iter[720] loss: [0.08750557154417038, 0.06141035258769989] \n",
      "epoch[39] iter[760] loss: [0.0687922015786171, 0.058865081518888474] \n",
      "Finish epoch[39], time elapsed[571.5257840156555]\n"
     ]
    }
   ],
   "source": [
    "loss_all = []\n",
    "for epoch in range(num_epochs):\n",
    "    LaneNet_model.train()\n",
    "    ts = time.time()\n",
    "    for iter, batch in enumerate(data_loader_train):\n",
    "        input_image = Variable(batch[0]).to(device)\n",
    "        binary_labels = Variable(batch[1]).to(device)\n",
    "        instance_labels = Variable(batch[2]).to(device)\n",
    "        \n",
    "        binary_final_logits, instance_embedding = LaneNet_model(input_image)\n",
    "        # loss = LaneNet_model.compute_loss(binary_logits=binary_final_logits, binary_labels=binary_labels,\n",
    "        #                               instance_logits=instance_embedding, instance_labels=instance_labels, delta_v=0.5, delta_d=3)\n",
    "        binary_segmenatation_loss, instance_segmenatation_loss = criterion(binary_logits=binary_final_logits, binary_labels=binary_labels,\n",
    "                                       instance_logits=instance_embedding, instance_labels=instance_labels, delta_v=0.5, delta_d=3)\n",
    "        \n",
    "        # binary_segmenatation_loss = criterion(binary_final_logits, binary_labels)\n",
    "        loss = 1*binary_segmenatation_loss + 1*instance_segmenatation_loss\n",
    "        optimizer.zero_grad()\n",
    "        loss_all.append(loss.item())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if iter % 40 == 0:\n",
    "            print(\"epoch[{}] iter[{}] loss: [{}, {}] \".format(epoch, iter, binary_segmenatation_loss.item(), instance_segmenatation_loss.item()))\n",
    "    lr_scheduler.step()\n",
    "    print(\"Finish epoch[{}], time elapsed[{}]\".format(epoch, time.time() - ts))\n",
    "    torch.save(LaneNet_model.state_dict(), \n",
    "                       f\"/TUSIMPLE/Lanenet_output/lanenet_epoch_{epoch}_batch_{8}_AUG.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f35791e7710>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAaSUlEQVR4nO3deZhV9Z3n8fe3NqDYCqTY1XJFkahgxXZJ22mXiIRRk7TzmJ50O2mnnWjSnThZHhxHY/dE48RMxixGY6IRgztKtIOCREUlrMUmhexQQLFVFVBUQVHr/c0f99R2b92Cukvd+ys+r+ep55469yzfX53Lh3N/ZzPnHCIi4p+sdBcgIiLxUYCLiHhKAS4i4ikFuIiIpxTgIiKeyunNlY0YMcIVFRX15ipFRLy3cuXKKudcYeT4Xg3woqIiSkpKenOVIiLeM7OdXY1XF4qIiKcU4CIinlKAi4h4SgEuIuIpBbiIiKcU4CIinlKAi4h4yosAn7O6nBeWdXkapIjIKcuLAH9rzV5eWbE73WWIiGSUEwa4mT1rZhVmVtph3HAzW2BmW4LXYSkt0oyQHjwhItLJyeyBPwdMjRg3A3jPOXce8F7we8qYQSiUyjWIiPjnhAHunPsIOBQx+hZgZjA8E7g1yXV1YmZo/1tEpLN4+8BHOef2AQSvI2NNaGZ3mVmJmZVUVlbGtTILryeueUVE+qqUH8R0zj3tnCt2zhUXFkbdDfGkZJmh/BYR6SzeAD9gZmMAgteK5JUULSsLHcQUEYkQb4C/BdwRDN8BvJmccrpm6CwUEZFIJ3Ma4UvAEmCCmZWb2Z3Ao8ANZrYFuCH4PWXM0EFMEZEIJ3wij3PuqzHeui7JtcRk6gMXEYnixZWYWaazUEREInkS4EZI+S0i0okXAW7oLBQRkUh+BLj6wEVEongS4OoDFxGJ5EWAZ+k0QhGRKJ4EuC7kERGJ5EWAm6GzUEREIngS4DqIKSIS6YRXYmaCF5ftSncJIiIZx4s9cBERiaYAFxHxlBcB/jfnx/cgCBGRvsyLAJ8wejD9c70oVUSk13iRiuFnYqa7ChGRzOJFgKMrMUVEongR4Fl6JI+ISBQvAly3kxURieZHgGsHXEQkih8Bjul2siIiEbwIcN1OVkQkmhcBjm5mJSISxYsAt+BV3SgiIu38CPAgwZXfIiLtvAjwrCDBld8iIu28CPDWLhSdCy4i0s6PAFcXiohIFE8CvLULRQkuItLKiwBvpT1wEZF2XgR460FMERFpl1CAm9m9ZrbezErN7CUz65+swjqvJ/yqg5giIu3iDnAzGwf8K1DsnJsEZAO3J6uwTusKXpXfIiLtEu1CyQEGmFkOkA/sTbykaG1noaRi4SIinoo7wJ1ze4CfAruAfcAR59y7kdOZ2V1mVmJmJZWVlfEV2XoWinbBRUTaJNKFMgy4BTgLGAsMNLOvRU7nnHvaOVfsnCsuLEzs6fIh5beISJtEulCuB3Y45yqdc03AG8BVySmrM1MfiohIlEQCfBdwhZnlWzhhrwM2JKesztoOYirBRUTaJNIHvgyYDawC1gXLejpJdXWSpUvpRUSi5CQys3Puh8APk1RLTLsPHwegoTmU6lWJiHjDiysxn1m0A4Cl2w+muRIRkczhRYC3Uh+4iEg7rwI8pB4UEZE2fgW4jmKKiLTxKsCV3yIi7fwKcPWBi4i08SrAdSm9iEg7zwJcCS4i0sqzAE93BSIimcOLAP9s0TAAzhien+ZKREQyhxcB/o9XFgEwdmhKntgmIuIlLwJczzQWEYnmRYC3Uhe4iEg7LwLc0C64iEgkLwK8lc4iFBFp50WAtz9RTQkuItLKjwBPdwEiIhnIiwBvpS4UEZF2XgS46ZmYIiJRvAhwdaKIiETzJMDDdBBTRKSdFwGuLhQRkWh+BHi6CxARyUBeBLiIiETzIsBNd7MSEYniRYC3Uh+4iEg7LwK8df9bZ6GIiLTzI8DVgyIiEsWLAG+lLhQRkXZeBHj73QhFRKRVQgFuZgVmNtvMNprZBjO7MlmFdVqPzgQXEYmSk+D8PwfmOef+zszygJQ+Nt6pD0VEpE3ce+BmNgS4BngGwDnX6JyrTlZhHVXWNgCwp/p4KhYvIuKlRLpQzgYqgd+b2Woz+52ZDYycyMzuMrMSMyuprKyMa0WzV5UDMGvpzgTKFRHpWxIJ8BxgCvCkc24ycAyYETmRc+5p51yxc664sLAwrhW1nQeuHhQRkTaJBHg5UO6cWxb8PptwoCed7kYoIhIt7gB3zu0HdpvZhGDUdcCnSakqQutZKLoSU0SkXaJnofwL8EJwBsp24OuJlxRNe+AiItESCnDn3BqgOEm1xKQLeUREovlxJWZrF4p2wUVE2vgR4NoDFxGJ4kWAi4hINC8CvPWJPOpBERFp50eAB6/qAxcRaedHgKsPXEQkih8BHrxqB1xEpJ0XAZ5luhJTRCSSFwGuKzFFRKJ5EeATxwzp9CoiIp4E+PUTRwEwddLoNFciIpI5vAjwVupCERFp50WA66HGIiLRvAjwVs2hULpLEBHJGF4EePXxRgAeeislz4sQEfGSFwF+tL4ZgP019WmuREQkc3gR4KYucBGRKJ4EuBJcRCSSHwGe7gJERDKQFwF+QH3fIiJRvAjwsoN16S5BRCTjeBHg6kIREYnmRYBnZSnCRUQieRHgOglFRCSaFwGerQQXEYniRYBnKcBFRKL4EeDqAxcRieJHgCu/RUSieBHgpw3MS3cJIiIZx4sAv/rcEekuQUQk4yQc4GaWbWarzexPySioK9nqQxERiZKMPfBvAxuSsJyYdBaKiEi0hALczMYDXwR+l5xyuqazUEREoiW6B/448AMg5sMqzewuMysxs5LKysq4VqL8FhGJFneAm9l0oMI5t7K76ZxzTzvnip1zxYWFhXGtS10oIiLREtkDvxq42czKgJeBa81sVlKqiqAAFxGJFneAO+fuc86Nd84VAbcD7zvnvpa0yjpQF4qISDQvzgPvl5ud7hJERDJOTjIW4pxbCCxMxrK6MqhfUsoUEelTvNgDFxGRaApwERFPKcBFRDylABcR8ZQCXETEUwpwERFPKcBFRDylABcR8ZR3AV7X2JzuEkREMoJ3Af7I2yl9doSIiDe8C/DDx5rSXYKISEbwLsBFRCTMuwA/3tSS7hJERDKCdwH+/saKdJcgIpIRvAtwEREJU4CLiHhKAS4i4ikFuIiIpxTgIiKeUoCLiHhKAS4i4ikFuIiIpxTgIiKeUoCLiHhKAS4i4ikFuIiIpxTgIiKe8jLAP9xcme4SRETSzssA/2R3dbpLEBFJOy8DXEREPA1wl+4CREQyQNwBbmanm9kHZrbBzNab2beTWVh3/rh6T2+tSkQkYyWyB94MfNc5dyFwBfBNM5uYnLKi5WZb2/D2qmPsPlRH0Yy5bNxfk6pViohktLgD3Dm3zzm3KhiuBTYA45JVWKRLxhd0+v3eV9YA8OqK8lStUkQkoyWlD9zMioDJwLIu3rvLzErMrKSyMv7T/8YWDOj0e8nOw3EvS0SkL0g4wM1sEPA68B3nXFR/hnPuaedcsXOuuLCwMO715Odldzne6ZCmiJyiEgpwM8slHN4vOOfeSE5JXfvuFyakcvEiIt5J5CwUA54BNjjnfpa8kro2fGBel+OddsBF5BSVyB741cA/ANea2ZrgZ1qS6oqSnWUnnkhE5BSSE++MzrlFgFJVRCRNvLwSU0RE+kiA1ze1UDRjLq+u2J3uUkREeo33Af7c4jKeWbQDgF+8vyXN1YiI9B7vAxzgsfmbAJ2RIiKnlj4R4CIip6I+FeBOu+AicgrpUwEuInIq6VMBvvdIPfVNLbSEtCcuIn1fnwpwgAsemMe/vrQ63WWIiKScVwEe634okeau25fiSkRE0s+rAH/j7qvSXYKISMbwKsCLRgxMdwkiIhnDqwAXEZF2fTbA3994IN0liIiklHcBfv2FI09qun96roSbf7WIkE4pFJE+yrsAf/z2ySc97SflR7jsRwtYV34khRWJiKSHdwE+qF/PnkFxuK6J//SrRTQ0tzCvNHx64YqyQxxvbElFeSIivca7AAe49/rzezzPT+Zt4huzVjFndTm3PbWECx+cF/PeKccbW9h9qC7RMkVEUsrLAP/Wtef2eJ5dQSDvPnS8bdyOqmOdpnl73T6WbDvIPz9fwl//5IPEihQRSTEvAzw7y7hp0ugezVPfFO4y+dmCzW3jXinp/ASfe15YxVd/u5RFW6t6tOzdh+r4eEtlj+aJdKCmnsXberZeETm1eRngAL/+L1O465qzT3r6j7dEh+NvPtzOG6vKE74N7TWPfcA/PLM8oWV88ReL+PvfLktoGV15b8MB1u6uTvpyRST9vA1wM2PSuKEJL+d/vLqWs+57m5r6pqj3pj7+EXWNzV3O97/+uI5/fr4EiO9JQK3fCFpVHW2IOe0P3yxlzupyQiEXNd+J3DmzhFue+EvPCxSRjOdtgAPceNEobr10bFKWdfFD70aN27i/lokPzueKR94D4J11+yiaMZez7pvLrKW7WPBp54uFimbMbQvi2vomXly2i482R3et/PnTA1zwwLwuT2+srmtk+Y5DncbNXLKTe19Zy4NvlXLBA/O6Pbe9obmFx+ZvjPkfTyzvrt9P0Yy5VNTW92g+n726YjdbK46muwyRuPXsnLwM0y8nm8dvn0zZwTrWpLCbYH9NPf/47PK2MO64x/23P13YadriH/2ZzxYNY0XZ4bZxf3fZeH562yXsP1LPql2HueeFVQA8v6SM11aW84c7L2+b9tJ/XwDAn/7lc1HfMGYt3RWzxpr6Jp5cuI3h+Xk88cE2ssz47hcmtL2/ouwQny0aHnP+PyzdCcDP3t3M1Emj+fyEE18wtWFfDTf9/GMWfu/z3d6n5n/OWcc5hYO483NnnXCZ8Vq/9wizlu7i4VsnkZVlJzXPD17/hOwsY9sj01JWl0gqeb0H3uqZO4pTvo6u9qQh+kwWoFN4A8xeWU7RjLlc8eP32sIb4LWV5QDc+8qaqGVM/+Ui5pXu63Jv++YnFrHlQC3NLSGWbDvI7z7ezhWPvMeTC7fx8NsbAKit77wHfttTS7ptn1k49F5esZv/+vsVwTLC3Up/2VrFu+v3U1nbwGcemk/pnvA3h9eD+t/9dH/M5W7aX8uLy3bxv//0abfrb24JUTRjLr//y45up4vljmdX8NLyXVQdi90V1RU9/EN85vUeeKvTBvVj1p1/xdeeSf5BwN5QdbSxy/HfmLWqy/Gle2q44f991O0yn1tcxnOLyzqNK5oxl+X3X8fIwf3bxrX2qR+M6IOfvbKc7722ljFD+7PvSLhb5cHpE6mtb2b6Lxex4d+nEmQ+Ly/fzbiCfL754ir++zVn861rz2Xm4jLu/vy53Ph4e51FM+Zyx5Vncu8N5/PCsl0s2XaQX/39ZAry8zge1PHT+Zv4+tVnsbXiKGOG9ue5xWX0y8li+sVjGTWkH80hR9XRBsYMHRD87Rqoa2gBwkG8+1AdIwf3xznHscYWvvfqWoqLhtEvJ4tJ44ZSureGB/5YypL7ru3273eymltCNIcc/XOzk7I8kZ6w3nwQcHFxsSspKUn5em57anHUXrD0vpwsozmOPdyHvzSJ++eURo2/+tzTONbQwprd1ZxdOJDtldHffgAWz7iWqx59v8v3Jo4Zwqf7arp87/W7r+L8UYMY3D+3bdyP39nAbz7czvZHpnXqmnHO8bMFm/nl+1sBKHv0i1HLq65rJCc7q8dXD4tEMrOVzrmoroY+GeCNzSH+sHQnd1x5JjPeWMfs4Ku+yMl6cPpEKmobeOrDbQB89P2/pX9eFs8uKuNLk8fx2PxN/HlD+0HsjgFeU9/U6aD4lodv4nhTC/1ysuiXE95T33Wwjv/4ZC/XXziKswsHkpudhXOO5pCjpOww89fv588bDvDz2y/lsjOjj138ZWsVRSMGcvBoA+MKBlCQn0d2h/9gKmrreXjuBn785c+Qn3fy/4G0hBwh58jNbu9dXb3rMEeON7UdFznW0MyDb67nK5eN46pzRgBw6Fgjw/Jz27riThUvLtvFA2+WsvlHN3X6+yfbKRXgkR6bv5EnPtjW6+sVORV9afI4BvbLZtP+Wr4yZTwfbank7XX7yc/LxoBjwX2I/u3mi1i6/SDvlLYfQ7n10rH8cc3ett+vOuc0Lj9rOOePGsyy7QdZW36E7ZVHqalv5ozh+TS3hNgbdPF9afI4auubmDhmCKcPz+f7sz8BoCA/l+q6Ju676QIO1DQwYfQgZi3dRdGIgfzH2r3cMHEU540cREvI8ZXLxlNd18R//k34mNEL/+2v+MaslYwa0p8f3TqJ6rpG5qzeQ1OLY+qk0fwgWMfdnz+HJxeGM2bed/4aw7h/zjqmXzyGM0cM5FhDM9Mvjv+MuVM6wDsKhRyNLSGaWkIs2lLFmacN5IE3S/nylHH8+oNt7Kk+zvdvnMCn+2o4Z8RABvfPbTswOO0zo3l7XewDdiIiscz/zjVMGD04rnlTEuBmNhX4OZAN/M4592h302dCgPeWUMjRFAq1fWUOBV9NN+6vxQwuGjuU440tHGtsprquCXCMH5ZP/9xsWkKO5TsOMXHsEA4ebeDQsUZ+vXAbnxk3lMlnFLBpfy0XjBnCx5srqTrawA0TR5Ofl82c1XvIyTLOHz2YR9/ZyLiCAVx6RgGnD8unoqaeN1bv4dZLx7Kl4ijr99Zw8yVjead0H00tjtsuG992VoyIJN+ce65i8hnD4po36QFuZtnAZuAGoBxYAXzVORfzfLFTKcDFP/VNLeRmZ9HUEgLodGbJsYZmcrOzyMkyzKChOURudhbZWUYo5Fi64yDnjRzM0AG5hFz4itm8nCyaQ46BeTl8uLmCS8YXsKPqGGMLBmAGWWY0NIX4aEsl544cxKRxQzla30zV0QbycrJYtuMQ54wYSMnOwxSNGMjk0wvYW32cw3WN7DxYxwVjhtASCjFz8U4+WzSMNbuPUFw0jGvOK2TxtioG5GUzekh/Vu+q5uOtVeyoPMqXp4znwjGDqahpoOpoAzOX7OQHUyewfk8N6/Yc4ZLTCygYkMun+2rYefAY5xQOYufBOs4bNYibJo3hF+9tYXD/HIqLhvHS8t1cXjSc5WXhC89GDMrj4LHGLq9Mzs/LZlh+HnuqjzNycD8qars+3XPogFyOHA+fvnr+qEFsPtB3LrRa+8MvMHRA7okn7EIqAvxK4CHn3I3B7/cBOOd+HGseBbiISM/FCvBELuQZB3S8nV95MC5yxXeZWYmZlVRWJnbHPhERaZdIgHd1zkzU7rxz7mnnXLFzrriwsDCB1YmISEeJBHg5cHqH38cDe2NMKyIiSZZIgK8AzjOzs8wsD7gdeCs5ZYmIyInEfY2vc67ZzL4FzCd8GuGzzrn1SatMRES6ldBNGpxzbwNvJ6kWERHpgT5xO1kRkVORAlxExFO9ei8UM6sEdsY5+wjA98e294U2QN9oh9qQOfpCO1LdhjOdc1HnYfdqgCfCzEq6uhLJJ32hDdA32qE2ZI6+0I50tUFdKCIinlKAi4h4yqcAfzrdBSRBX2gD9I12qA2Zoy+0Iy1t8KYPXEREOvNpD1xERDpQgIuIeMqLADezqWa2ycy2mtmMdNcTyczKzGydma0xs5Jg3HAzW2BmW4LXYcF4M7NfBG35xMymdFjOHcH0W8zsjhTX/KyZVZhZaYdxSavZzC4L/iZbg3mT/sjuGG14yMz2BNtijZlN6/DefUE9m8zsxg7ju/x8BTdqWxa07ZXgpm3JbsPpZvaBmW0ws/Vm9u1gvG/bIlY7vNkeZtbfzJab2dqgDf/W3XrNrF/w+9bg/aJ42xY351xG/xC+UdY24GwgD1gLTEx3XRE1lgEjIsb9BJgRDM8A/k8wPA14h/D91K8AlgXjhwPbg9dhwfCwFNZ8DTAFKE1FzcBy4MpgnneAm3qpDQ8B3+ti2onBZ6cfcFbwmcru7vMFvArcHgw/BdydgjaMAaYEw4MJP6ZwoofbIlY7vNkewd9nUDCcCywL/sZdrhe4B3gqGL4deCXetsX748Me+OXAVufcdudcI/AycEuaazoZtwAzg+GZwK0dxj/vwpYCBWY2BrgRWOCcO+ScOwwsAKamqjjn3EfAoVTUHLw3xDm3xIU/0c93WFaq2xDLLcDLzrkG59wOYCvhz1aXn69gL/VaYHYwf8e/R9I45/Y551YFw7XABsJPtvJtW8RqRywZtz2Cv2nrQzhzgx/XzXo7bqPZwHVBnT1qWyI1+xDgJ/XotjRzwLtmttLM7grGjXLO7YPwhxsYGYyP1Z5MaGeyah4XDEeO7y3fCroXnm3teqDnbTgNqHbONUeMT5ngK/hkwnt+3m6LiHaAR9vDzLLNbA1QQfg/wW3drLet1uD9I0GdvfZv3IcAP6lHt6XZ1c65KcBNwDfN7Jpupo3VnkxuZ09rTmdbngTOAS4F9gH/Nxif0W0ws0HA68B3nHM13U3axbhMbodX28M51+Kcu5TwE8YuBy7sZr1pb4MPAZ7xj25zzu0NXiuAOYQ3/IHg6yvBa0Uweaz2ZEI7k1VzeTAcOT7lnHMHgn+EIeC3hLcFJ6i1q/FVhLsnciLGJ52Z5RIOvRecc28Eo73bFl21w8ftEdRdDSwk3Acea71ttQbvDyXcpdd7/8aTeRAgFT+EHzqxnfDBgNaO/4vSXVeH+gYCgzsMLybcd/0YnQ9C/SQY/iKdD0ItD8YPB3YQPgA1LBgenuLai+h8ADBpNRN+5N4VtB84m9ZLbRjTYfhewn2RABfR+cDSdsIHlWJ+voDX6Hzw6p4U1G+E+6Ufjxjv1bboph3ebA+gECgIhgcAHwPTY60X+CadD2K+Gm/b4q45Ff+oUvDhmEb4qPY24P501xNR29nBhlgLrG+tj3Bf2HvAluC19R+TAU8EbVkHFHdY1j8RPuCxFfh6iut+ifBX2ibCewZ3JrNmoBgoDeb5FcFVv73Qhj8ENX5C+BmtHQPk/qCeTXQ4EyPW5yvYtsuDtr0G9EtBGz5H+Gv0J8Ca4Geah9siVju82R7AxcDqoNZS4MHu1gv0D37fGrx/drxti/dHl9KLiHjKhz5wERHpggJcRMRTCnAREU8pwEVEPKUAFxHxlAJcRMRTCnAREU/9fx24jvOs5bAnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pylab as plt\n",
    "plt.plot(loss_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
